You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2015/04/01 15:18:58 UTC

[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2871 - Still Failing

Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2871/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:28021/jx_pe/gu/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28021/jx_pe/gu/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([634A0386B04539CD:EB1E3C5C1EB95435]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28336/jt_tm/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28336/jt_tm/collection1
	at __randomizedtesting.SeedInfo.seed([634A0386B04539CD:EB1E3C5C1EB95435]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28336/jt_tm/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:41932/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:41932/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([634A0386B04539CD:EB1E3C5C1EB95435]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:41932/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9626 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/init-core-data-001
   [junit4]   2> 364503 T1514 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 364508 T1514 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 364508 T1515 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 364608 T1514 oasc.ZkTestServer.run start zk server on port:41901
   [junit4]   2> 364609 T1514 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 364610 T1514 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 364613 T1522 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6133293f name:ZooKeeperConnection Watcher:127.0.0.1:41901 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 364613 T1514 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 364614 T1514 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 364614 T1514 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 364616 T1514 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 364617 T1514 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 364618 T1525 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2ebd36bd name:ZooKeeperConnection Watcher:127.0.0.1:41901/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 364618 T1514 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 364618 T1514 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 364618 T1514 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 364620 T1514 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 364621 T1514 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 364622 T1514 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 364623 T1514 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 364624 T1514 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 364626 T1514 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 364626 T1514 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 364628 T1514 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 364629 T1514 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 364630 T1514 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 364630 T1514 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 364632 T1514 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 364632 T1514 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 364633 T1514 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 364634 T1514 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 364635 T1514 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 364635 T1514 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 364637 T1514 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 364637 T1514 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 364639 T1514 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 364639 T1514 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 364640 T1514 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 364641 T1514 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 364642 T1514 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 364642 T1514 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 365603 T1514 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/cores/collection1
   [junit4]   2> 365606 T1514 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 365610 T1514 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:41912
   [junit4]   2> 365610 T1514 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=41911, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/cores}
   [junit4]   2> 365611 T1514 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 365611 T1514 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/'
   [junit4]   2> 365644 T1514 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/solr.xml
   [junit4]   2> 365692 T1514 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/cores
   [junit4]   2> 365692 T1514 oasc.CoreContainer.<init> New CoreContainer 2074659216
   [junit4]   2> 365692 T1514 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/]
   [junit4]   2> 365693 T1514 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/lib
   [junit4]   2> 365693 T1514 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/lib).
   [junit4]   2> 365702 T1514 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 365702 T1514 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 365702 T1514 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 365703 T1514 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 365703 T1514 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 365703 T1514 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 365703 T1514 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 365703 T1514 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 365704 T1514 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 365704 T1514 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 365704 T1514 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 365705 T1514 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 365705 T1514 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 365705 T1514 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 365705 T1514 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 365706 T1514 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:41901/solr
   [junit4]   2> 365706 T1514 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 365706 T1514 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 365707 T1514 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 365709 T1537 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6142d975 name:ZooKeeperConnection Watcher:127.0.0.1:41901 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 365709 T1514 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 365710 T1514 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 365712 T1514 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 365713 T1540 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@475802dc name:ZooKeeperConnection Watcher:127.0.0.1:41901/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 365713 T1514 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 365714 T1514 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 365716 T1514 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 365718 T1514 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 365719 T1514 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 365721 T1514 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 365723 T1514 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 365724 T1514 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 365725 T1514 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 365726 T1514 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41911_
   [junit4]   2> 365727 T1514 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:41911_
   [junit4]   2> 365728 T1514 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 365729 T1514 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 365731 T1514 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 365732 T1514 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:41911_
   [junit4]   2> 365732 T1514 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 365733 T1514 oasc.Overseer.start Overseer (id=93578256990404611-127.0.0.1:41911_-n_0000000000) starting
   [junit4]   2> 365735 T1514 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 365740 T1514 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 365741 T1542 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 365741 T1514 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 365741 T1541 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 365744 T1514 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/cores
   [junit4]   2> 365746 T1514 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {transient=false, dataDir=data/, shard=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/cores/collection1/, coreNodeName=, schema=schema.xml, collection=control_collection, name=collection1, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/cores/collection1, loadOnStartup=true}
   [junit4]   2> 365746 T1514 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/cores/collection1/
   [junit4]   2> 365747 T1514 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 365747 T1544 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 365747 T1544 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 365748 T1544 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 365748 T1540 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 365749 T1541 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41911",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:41911_"} current state version: 0
   [junit4]   2> 365749 T1541 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41911",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:41911_"}
   [junit4]   2> 365750 T1541 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 365750 T1541 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 365751 T1540 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 366748 T1544 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 366749 T1544 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 366749 T1544 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 366750 T1544 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 366750 T1544 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 366751 T1544 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/cores/collection1/'
   [junit4]   2> 366767 T1544 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 366770 T1544 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 366780 T1544 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 366803 T1544 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 366852 T1544 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 366854 T1544 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 366865 T1544 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 367309 T1544 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 367326 T1544 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 367329 T1544 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 367344 T1544 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 367349 T1544 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 367353 T1544 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 367353 T1544 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 367354 T1544 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 367354 T1544 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 367355 T1544 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 367355 T1544 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 367355 T1544 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 367356 T1544 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 367356 T1544 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/cores/collection1/data/
   [junit4]   2> 367356 T1544 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5143bf4
   [junit4]   2> 367357 T1544 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/cores/collection1/data
   [junit4]   2> 367358 T1544 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/cores/collection1/data/index/
   [junit4]   2> 367358 T1544 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 367358 T1544 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/cores/collection1/data/index
   [junit4]   2> 367359 T1544 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=10, maxMergeAtOnceExplicit=47, maxMergedSegmentMB=12.9931640625, floorSegmentMB=0.5966796875, forceMergeDeletesPctAllowed=21.044805038717328, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 367360 T1544 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6718d7c5 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@ffdfb64),segFN=segments_1,generation=1}
   [junit4]   2> 367360 T1544 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 367367 T1544 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 367367 T1544 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 367368 T1544 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 367368 T1544 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 367368 T1544 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 367368 T1544 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 367369 T1544 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 367369 T1544 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 367369 T1544 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 367388 T1544 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 367391 T1544 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 367393 T1544 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 367395 T1544 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 367401 T1544 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/logging,/admin/luke,/admin/ping,/get,/update/json/docs,/config,/admin/segments,/update/csv,/admin/mbeans,/update/json,/admin/system,/update,/admin/properties,/admin/plugins,/replication,standard,/admin/file,/schema,/admin/threads
   [junit4]   2> 367403 T1544 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 367405 T1544 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 367405 T1544 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 367407 T1544 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 367407 T1544 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 367408 T1544 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=15, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=61.974609375, floorSegmentMB=0.5615234375, forceMergeDeletesPctAllowed=4.815556489907208, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.445132059291366
   [junit4]   2> 367408 T1544 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6718d7c5 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@ffdfb64),segFN=segments_1,generation=1}
   [junit4]   2> 367409 T1544 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 367409 T1544 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@47d821bf[collection1] main
   [junit4]   2> 367409 T1544 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 367410 T1544 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 367410 T1544 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 367411 T1544 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 367411 T1544 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 367411 T1544 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 367412 T1544 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 367412 T1544 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 367412 T1544 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 367413 T1544 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 367414 T1545 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@47d821bf[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 367414 T1544 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 367415 T1548 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:41911 collection:control_collection shard:shard1
   [junit4]   2> 367415 T1514 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 367416 T1514 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 367416 T1548 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 367419 T1514 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 367419 T1514 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 367421 T1552 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6c3b7305 name:ZooKeeperConnection Watcher:127.0.0.1:41901/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 367421 T1548 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 367421 T1514 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 367422 T1514 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 367422 T1514 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 367423 T1540 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 367423 T1548 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 367423 T1548 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C220 name=collection1 org.apache.solr.core.SolrCore@7d794040 url=http://127.0.0.1:41911/collection1 node=127.0.0.1:41911_ C220_STATE=coll:control_collection core:collection1 props:{state=down, core=collection1, base_url=http://127.0.0.1:41911, node_name=127.0.0.1:41911_}
   [junit4]   2> 367423 T1548 C:control_collection S:shard1 C220 P41911 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:41911/collection1/
   [junit4]   2> 367423 T1541 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 367424 T1548 C:control_collection S:shard1 C220 P41911 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 367424 T1548 C:control_collection S:shard1 C220 P41911 oasc.SyncStrategy.syncToMe http://127.0.0.1:41911/collection1/ has no replicas
   [junit4]   2> 367424 T1548 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:41911/collection1/ shard1
   [junit4]   2> 367425 T1548 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 367425 T1514 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 367425 T1552 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 367425 T1540 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 367429 T1540 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 367429 T1541 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41911",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 367532 T1540 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 367532 T1552 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 367581 T1548 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:41911/collection1/ and leader is http://127.0.0.1:41911/collection1/
   [junit4]   2> 367582 T1548 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:41911
   [junit4]   2> 367582 T1548 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 367582 T1548 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 367583 T1548 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 367584 T1540 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 367585 T1541 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41911",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:41911_"} current state version: 3
   [junit4]   2> 367586 T1541 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41911",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:41911_"}
   [junit4]   2> 367689 T1552 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 367689 T1540 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 368343 T1514 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/cores/collection1
   [junit4]   2> 368346 T1514 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001
   [junit4]   2> 368347 T1514 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 368349 T1514 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:41929
   [junit4]   2> 368349 T1514 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, hostPort=41928, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/tempDir-001/jetty1, solrconfig=solrconfig.xml}
   [junit4]   2> 368350 T1514 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 368350 T1514 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/'
   [junit4]   2> 368379 T1514 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/solr.xml
   [junit4]   2> 368424 T1514 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/cores
   [junit4]   2> 368424 T1514 oasc.CoreContainer.<init> New CoreContainer 1311768164
   [junit4]   2> 368425 T1514 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/]
   [junit4]   2> 368425 T1514 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/lib
   [junit4]   2> 368426 T1514 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/lib).
   [junit4]   2> 368438 T1514 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 368439 T1514 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 368439 T1514 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 368439 T1514 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 368440 T1514 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 368440 T1514 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 368440 T1514 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 368441 T1514 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 368441 T1514 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 368441 T1514 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 368442 T1514 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 368442 T1514 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 368443 T1514 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 368443 T1514 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 368443 T1514 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 368444 T1514 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:41901/solr
   [junit4]   2> 368444 T1514 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 368444 T1514 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 368445 T1514 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 368447 T1564 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3dbf589a name:ZooKeeperConnection Watcher:127.0.0.1:41901 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 368447 T1514 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 368448 T1514 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 368450 T1514 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 368452 T1567 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6745ac53 name:ZooKeeperConnection Watcher:127.0.0.1:41901/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 368452 T1514 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 368457 T1514 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 369461 T1514 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41928_
   [junit4]   2> 369461 T1514 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:41928_
   [junit4]   2> 369464 T1514 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 369466 T1514 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/cores
   [junit4]   2> 369468 T1514 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {transient=false, config=solrconfig.xml, collection=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/cores/collection1, dataDir=data/, coreNodeName=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/cores/collection1/, name=collection1, schema=schema.xml, loadOnStartup=true, shard=}
   [junit4]   2> 369468 T1514 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/cores/collection1/
   [junit4]   2> 369468 T1514 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 369469 T1568 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 369469 T1568 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 369470 T1540 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 369470 T1568 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 369471 T1541 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41928",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:41928_"} current state version: 4
   [junit4]   2> 369472 T1541 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41928",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:41928_"}
   [junit4]   2> 369472 T1541 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 369472 T1541 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 369473 T1567 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 369473 T1552 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 369473 T1540 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 370471 T1568 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 370472 T1568 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 370472 T1568 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 370473 T1568 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 370473 T1568 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 370474 T1568 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/cores/collection1/'
   [junit4]   2> 370490 T1568 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 370493 T1568 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 370503 T1568 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 370525 T1568 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 370573 T1568 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 370575 T1568 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 370586 T1568 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 370918 T1568 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 370930 T1568 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 370932 T1568 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 370943 T1568 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 370946 T1568 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 370948 T1568 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 370949 T1568 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 370949 T1568 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 370949 T1568 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 370950 T1568 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 370950 T1568 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 370950 T1568 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 370951 T1568 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 370951 T1568 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 370951 T1568 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5143bf4
   [junit4]   2> 370952 T1568 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/cores/collection1/data
   [junit4]   2> 370952 T1568 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 370952 T1568 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 370952 T1568 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 370953 T1568 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=10, maxMergeAtOnceExplicit=47, maxMergedSegmentMB=12.9931640625, floorSegmentMB=0.5966796875, forceMergeDeletesPctAllowed=21.044805038717328, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 370953 T1568 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1bfbd325 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@31e3307d),segFN=segments_1,generation=1}
   [junit4]   2> 370954 T1568 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 370959 T1568 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 370959 T1568 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 370960 T1568 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 370960 T1568 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 370960 T1568 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 370960 T1568 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 370961 T1568 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 370961 T1568 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 370961 T1568 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 370973 T1568 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 370975 T1568 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 370976 T1568 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 370978 T1568 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 370984 T1568 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/logging,/admin/luke,/admin/ping,/get,/update/json/docs,/config,/admin/segments,/update/csv,/admin/mbeans,/update/json,/admin/system,/update,/admin/properties,/admin/plugins,/replication,standard,/admin/file,/schema,/admin/threads
   [junit4]   2> 370985 T1568 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 370987 T1568 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 370987 T1568 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 370988 T1568 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 370988 T1568 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 370989 T1568 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=15, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=61.974609375, floorSegmentMB=0.5615234375, forceMergeDeletesPctAllowed=4.815556489907208, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.445132059291366
   [junit4]   2> 370990 T1568 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1bfbd325 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@31e3307d),segFN=segments_1,generation=1}
   [junit4]   2> 370990 T1568 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 370990 T1568 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@27f027d1[collection1] main
   [junit4]   2> 370990 T1568 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 370991 T1568 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 370991 T1568 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 370992 T1568 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 370992 T1568 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 370992 T1568 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 370992 T1568 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 370993 T1568 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 370993 T1568 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 370993 T1568 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 370994 T1569 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@27f027d1[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 370995 T1568 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 370995 T1572 C:collection1 S:shard2 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:41928 collection:collection1 shard:shard2
   [junit4]   2> 370996 T1514 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 370996 T1514 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 370996 T1572 C:collection1 S:shard2 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 371000 T1572 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 371001 T1540 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 371001 T1572 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 371001 T1572 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C221 name=collection1 org.apache.solr.core.SolrCore@78cbe4af url=http://127.0.0.1:41928/collection1 node=127.0.0.1:41928_ C221_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, base_url=http://127.0.0.1:41928, node_name=127.0.0.1:41928_}
   [junit4]   2> 371002 T1572 C:collection1 S:shard2 C221 P41928 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:41928/collection1/
   [junit4]   2> 371002 T1541 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 371002 T1572 C:collection1 S:shard2 C221 P41928 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 371003 T1572 C:collection1 S:shard2 C221 P41928 oasc.SyncStrategy.syncToMe http://127.0.0.1:41928/collection1/ has no replicas
   [junit4]   2> 371003 T1572 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:41928/collection1/ shard2
   [junit4]   2> 371003 T1572 C:collection1 S:shard2 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 371004 T1540 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 371004 T1552 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 371004 T1567 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 371007 T1540 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 371007 T1541 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41928",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 371110 T1540 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 371110 T1552 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 371111 T1567 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 371159 T1572 C:collection1 S:shard2 oasc.ZkController.register We are http://127.0.0.1:41928/collection1/ and leader is http://127.0.0.1:41928/collection1/
   [junit4]   2> 371160 T1572 C:collection1 S:shard2 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:41928
   [junit4]   2> 371160 T1572 C:collection1 S:shard2 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 371160 T1572 C:collection1 S:shard2 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 371160 T1572 C:collection1 S:shard2 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 371162 T1540 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 371162 T1541 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41928",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:41928_"} current state version: 7
   [junit4]   2> 371163 T1541 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41928",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:41928_"}
   [junit4]   2> 371266 T1552 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 371266 T1540 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 371266 T1567 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataCha

[...truncated too long message...]

index.20150401144121430
   [junit4]   2> 1611851 T7122 C:control_collection S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/jt_tm,null}
   [junit4]   2> 1612028 T7122 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:28319 28319
   [junit4]   2> 1612381 T7123 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:28319 28319
   [junit4]   2> 1612382 T7123 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1612382 T7122 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:28325/jt_tm
   [junit4]   2> 1612382 T7122 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:28354/jt_tm
   [junit4]   2> 1612383 T7122 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 6 connections to: http://127.0.0.1:28336/jt_tm
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=634A0386B04539CD -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ar_SY -Dtests.timezone=Africa/Kigali -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   71.0s J1 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28336/jt_tm/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([634A0386B04539CD:EB1E3C5C1EB95435]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28336/jt_tm/collection1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 54 more
   [junit4]   2> 1612404 T7122 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 634A0386B04539CD-001
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=ar_SY, timezone=Africa/Kigali
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=227733656,total=382730240
   [junit4]   2> NOTE: All tests run in this JVM: [TestPerFieldSimilarity, SuggesterTSTTest, TestNamedUpdateProcessors, DocValuesTest, MergeStrategyTest, HdfsChaosMonkeySafeLeaderTest, TestSchemaVersionResource, TestSchemaSimilarityResource, LoggingHandlerTest, NotRequiredUniqueKeyTest, SolrTestCaseJ4Test, DateFieldTest, StandardRequestHandlerTest, MinimalSchemaTest, AnalyticsMergeStrategyTest, AnalyticsQueryTest, TestIntervalFaceting, TestFiltering, SortByFunctionTest, ClusterStateTest, DistributedFacetPivotLongTailTest, DistributedSpellCheckComponentTest, TestBinaryField, AutoCommitTest, TestJmxIntegration, DisMaxRequestHandlerTest, TestUniqueKeyFieldResource, MoreLikeThisHandlerTest, CoreAdminCreateDiscoverTest, TestSystemIdResolver, TestSolrQueryParserDefaultOperatorResource, TestFoldingMultitermQuery, TestMaxScoreQueryParser, IndexSchemaTest, SpatialFilterTest, ChangedSchemaMergeTest, TestRestManager, SegmentsInfoRequestHandlerTest, SolrIndexConfigTest, TestFastLRUCache, DistributedFacetPivotWhiteBoxTest, TestJoin, TestManagedResourceStorage, TestSolrConfigHandlerConcurrent, SoftAutoCommitTest, TestFieldCollectionResource, OutputWriterTest, DeleteShardTest, DistribCursorPagingTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, SampleTest, BasicDistributedZkTest, AliasIntegrationTest, SpellingQueryConverterTest, DistanceUnitsTest, PathHierarchyTokenizerFactoryTest, SuggesterTest, LeaderFailoverAfterPartitionTest, TestReplicationHandler, TestCloudSchemaless, TestSolrConfigHandlerCloud, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J1 in 71.01s, 1 test, 1 error <<< FAILURES!

[...truncated 709 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 53 minutes 58 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 211947746 bytes
Compression is 0.0%
Took 41 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2918 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2918/

1 tests failed.
REGRESSION:  org.apache.solr.cloud.RecoveryZkTest.test

Error Message:
shard1 is not consistent.  Got 963 from http://127.0.0.1:32047/esw/zr/collection1lastClient and got 272 from http://127.0.0.1:32051/esw/zr/collection1

Stack Trace:
java.lang.AssertionError: shard1 is not consistent.  Got 963 from http://127.0.0.1:32047/esw/zr/collection1lastClient and got 272 from http://127.0.0.1:32051/esw/zr/collection1
	at __randomizedtesting.SeedInfo.seed([CED94C4AA2172441:468D73900CEB49B9]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:123)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 9725 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryZkTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/init-core-data-001
   [junit4]   2> 648134 T2610 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 648134 T2610 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /esw/zr
   [junit4]   2> 648141 T2610 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 648141 T2611 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 648241 T2610 oasc.ZkTestServer.run start zk server on port:32033
   [junit4]   2> 648242 T2610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 648243 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 648247 T2618 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3d31f642 name:ZooKeeperConnection Watcher:127.0.0.1:32033 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 648247 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 648248 T2610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 648248 T2610 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 648251 T2610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 648252 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 648254 T2621 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@50c588 name:ZooKeeperConnection Watcher:127.0.0.1:32033/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 648254 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 648255 T2610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 648255 T2610 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 648257 T2610 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 648259 T2610 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 648260 T2610 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 648261 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 648262 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 648265 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 648266 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 648268 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 648268 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 648270 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 648271 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 648273 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 648273 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 648275 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 648276 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 648277 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 648278 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 648280 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 648281 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 648282 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 648283 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 648285 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 648285 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 648287 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 648288 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 649512 T2610 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores/collection1
   [junit4]   2> 649515 T2610 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 649517 T2610 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:32039
   [junit4]   2> 649518 T2610 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=32039, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores, hostContext=/esw/zr}
   [junit4]   2> 649518 T2610 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 649518 T2610 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/'
   [junit4]   2> 649548 T2610 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/solr.xml
   [junit4]   2> 649583 T2610 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores
   [junit4]   2> 649583 T2610 oasc.CoreContainer.<init> New CoreContainer 235084862
   [junit4]   2> 649584 T2610 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/]
   [junit4]   2> 649584 T2610 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/lib
   [junit4]   2> 649584 T2610 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/lib).
   [junit4]   2> 649594 T2610 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 649594 T2610 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 649594 T2610 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 649595 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 649595 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 649595 T2610 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 649595 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 649595 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 649596 T2610 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 649596 T2610 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 649596 T2610 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 649601 T2610 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 649602 T2610 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 649602 T2610 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 649602 T2610 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 649602 T2610 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:32033/solr
   [junit4]   2> 649603 T2610 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 649603 T2610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 649603 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 649605 T2633 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@64a4870 name:ZooKeeperConnection Watcher:127.0.0.1:32033 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 649606 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 649606 T2610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 649608 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 649609 T2636 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@74862f9b name:ZooKeeperConnection Watcher:127.0.0.1:32033/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 649609 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 649611 T2610 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 649612 T2610 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 649613 T2610 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 649615 T2610 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 649616 T2610 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 649617 T2610 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 649618 T2610 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 649619 T2610 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 649620 T2610 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32039_esw%2Fzr
   [junit4]   2> 649621 T2610 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:32039_esw%2Fzr
   [junit4]   2> 649622 T2610 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 649623 T2610 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 649624 T2610 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 649625 T2610 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:32039_esw%2Fzr
   [junit4]   2> 649625 T2610 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 649626 T2610 oasc.Overseer.start Overseer (id=93624945334288387-127.0.0.1:32039_esw%2Fzr-n_0000000000) starting
   [junit4]   2> 649627 T2610 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 649631 T2610 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 649632 T2638 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 649632 T2637 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 649632 T2610 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 649635 T2610 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores
   [junit4]   2> 649636 T2610 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {transient=false, collection=control_collection, coreNodeName=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores/collection1/, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores/collection1, shard=, config=solrconfig.xml, dataDir=data/, loadOnStartup=true, name=collection1}
   [junit4]   2> 649637 T2610 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores/collection1/
   [junit4]   2> 649637 T2610 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 649637 T2640 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 649638 T2640 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 649638 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 649638 T2640 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 649639 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32039/esw/zr",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:32039_esw%2Fzr"} current state version: 0
   [junit4]   2> 649639 T2637 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32039/esw/zr",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:32039_esw%2Fzr"}
   [junit4]   2> 649639 T2637 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 649640 T2637 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 649641 T2636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 650639 T2640 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 650640 T2640 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 650640 T2640 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 650641 T2640 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 650641 T2640 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 650641 T2640 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores/collection1/'
   [junit4]   2> 650659 T2640 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 650661 T2640 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 650671 T2640 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 650692 T2640 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 650740 T2640 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 650741 T2640 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 650750 T2640 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 651106 T2640 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 651109 T2640 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 651111 T2640 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 651116 T2640 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 651136 T2640 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 651136 T2640 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 651137 T2640 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores/collection1/data/
   [junit4]   2> 651137 T2640 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@74cee5ed
   [junit4]   2> 651138 T2640 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores/collection1/data
   [junit4]   2> 651138 T2640 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores/collection1/data/index/
   [junit4]   2> 651138 T2640 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 651139 T2640 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores/collection1/data/index
   [junit4]   2> 651139 T2640 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=10, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 651140 T2640 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@569f34ac lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6e3b34a1),segFN=segments_1,generation=1}
   [junit4]   2> 651141 T2640 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 651146 T2640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 651147 T2640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 651147 T2640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 651147 T2640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 651148 T2640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 651148 T2640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 651148 T2640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 651149 T2640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 651149 T2640 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 651171 T2640 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 651173 T2640 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 651175 T2640 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 651178 T2640 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 651183 T2640 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/segments,/schema,/admin/properties,/update/csv,/admin/file,/admin/mbeans,/config,/admin/luke,/admin/plugins,/admin/ping,/admin/threads,/update/json,/update/json/docs,/update,/get,/admin/system,/replication,standard,/admin/logging
   [junit4]   2> 651185 T2640 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 651187 T2640 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 651187 T2640 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 651189 T2640 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 651189 T2640 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 651190 T2640 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=46, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.15818545914049942]
   [junit4]   2> 651190 T2640 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@569f34ac lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6e3b34a1),segFN=segments_1,generation=1}
   [junit4]   2> 651191 T2640 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 651191 T2640 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@6f7491f8[collection1] main
   [junit4]   2> 651191 T2640 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 651192 T2640 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 651192 T2640 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 651193 T2640 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 651193 T2640 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 651193 T2640 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 651194 T2640 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 651194 T2640 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 651194 T2640 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 651195 T2640 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 651195 T2641 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6f7491f8[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 651196 T2640 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 651197 T2644 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:32039/esw/zr collection:control_collection shard:shard1
   [junit4]   2> 651197 T2610 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 651197 T2610 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 651198 T2644 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 651200 T2610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 651200 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 651202 T2644 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 651202 T2647 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6a411a6a name:ZooKeeperConnection Watcher:127.0.0.1:32033/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 651202 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 651203 T2610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 651203 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 651203 T2644 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 651204 T2644 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 651203 T2610 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 651204 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C1808 name=collection1 org.apache.solr.core.SolrCore@43fe35d1 url=http://127.0.0.1:32039/esw/zr/collection1 node=127.0.0.1:32039_esw%2Fzr C1808_STATE=coll:control_collection core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:32039/esw/zr, node_name=127.0.0.1:32039_esw%2Fzr}
   [junit4]   2> 651204 T2644 C:control_collection S:shard1 C1808 P32039 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:32039/esw/zr/collection1/
   [junit4]   2> 651206 T2644 C:control_collection S:shard1 C1808 P32039 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 651206 T2644 C:control_collection S:shard1 C1808 P32039 oasc.SyncStrategy.syncToMe http://127.0.0.1:32039/esw/zr/collection1/ has no replicas
   [junit4]   2> 651206 T2644 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:32039/esw/zr/collection1/ shard1
   [junit4]   2> 651207 T2644 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 651207 T2636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 651207 T2647 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 651208 T2610 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 651208 T2610 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 651209 T2610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 651209 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 651211 T2650 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@762874b2 name:ZooKeeperConnection Watcher:127.0.0.1:32033/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 651211 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 651212 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 651212 T2610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 651213 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32039/esw/zr",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 651215 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 651216 T2637 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 651216 T2636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 651216 T2647 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 651219 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 651220 T2637 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 651221 T2647 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 651221 T2636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 651263 T2644 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:32039/esw/zr/collection1/ and leader is http://127.0.0.1:32039/esw/zr/collection1/
   [junit4]   2> 651264 T2644 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:32039/esw/zr
   [junit4]   2> 651264 T2644 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 651264 T2644 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 651264 T2644 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 651265 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 651266 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32039/esw/zr",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:32039_esw%2Fzr"} current state version: 4
   [junit4]   2> 651267 T2637 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32039/esw/zr",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:32039_esw%2Fzr"}
   [junit4]   2> 651370 T2636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 651370 T2647 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 652802 T2610 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores/collection1
   [junit4]   2> 652804 T2610 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001
   [junit4]   2> 652805 T2610 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 652808 T2610 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:32047
   [junit4]   2> 652808 T2610 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores, hostContext=/esw/zr, hostPort=32047, solrconfig=solrconfig.xml}
   [junit4]   2> 652808 T2610 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 652808 T2610 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/'
   [junit4]   2> 652838 T2610 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/solr.xml
   [junit4]   2> 652872 T2610 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores
   [junit4]   2> 652873 T2610 oasc.CoreContainer.<init> New CoreContainer 1177738588
   [junit4]   2> 652873 T2610 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/]
   [junit4]   2> 652873 T2610 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/lib
   [junit4]   2> 652874 T2610 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/lib).
   [junit4]   2> 652883 T2610 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 652883 T2610 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 652883 T2610 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 652884 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 652884 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 652884 T2610 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 652884 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 652884 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 652885 T2610 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 652885 T2610 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 652885 T2610 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 652889 T2610 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 652890 T2610 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 652890 T2610 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 652891 T2610 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 652891 T2610 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:32033/solr
   [junit4]   2> 652891 T2610 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 652891 T2610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 652892 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 652894 T2662 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2dff8c1a name:ZooKeeperConnection Watcher:127.0.0.1:32033 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 652894 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 652895 T2610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 652896 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 652897 T2665 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4028f3af name:ZooKeeperConnection Watcher:127.0.0.1:32033/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 652897 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 652901 T2610 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 653904 T2610 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32047_esw%2Fzr
   [junit4]   2> 653905 T2610 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:32047_esw%2Fzr
   [junit4]   2> 653909 T2610 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 653911 T2610 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores
   [junit4]   2> 653915 T2610 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores/collection1/, dataDir=data/, loadOnStartup=true, coreNodeName=, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores/collection1, collection=collection1, shard=, schema=schema.xml, transient=false, name=collection1}
   [junit4]   2> 653916 T2610 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores/collection1/
   [junit4]   2> 653916 T2610 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 653918 T2666 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 653918 T2666 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 653919 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 653920 T2666 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 653920 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32047/esw/zr",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:32047_esw%2Fzr"} current state version: 5
   [junit4]   2> 653921 T2666 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 653922 T2637 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32047/esw/zr",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:32047_esw%2Fzr"}
   [junit4]   2> 653922 T2666 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 653922 T2637 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 653922 T2637 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 653923 T2666 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 653923 T2666 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 653923 T2637 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 653924 T2665 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 653924 T2665 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 654923 T2666 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 654924 T2666 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 654924 T2666 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 654925 T2666 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 654925 T2666 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 654926 T2666 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores/collection1/'
   [junit4]   2> 654944 T2666 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 654947 T2666 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 654958 T2666 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 654982 T2666 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 655050 T2666 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 655052 T2666 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 655062 T2666 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 655423 T2666 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 655426 T2666 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 655428 T2666 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 655432 T2666 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 655451 T2666 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 655452 T2666 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 655452 T2666 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 655453 T2666 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@74cee5ed
   [junit4]   2> 655454 T2666 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores/collection1/data
   [junit4]   2> 655454 T2666 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 655454 T2666 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 655455 T2666 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 655455 T2666 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=10, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 655456 T2666 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5c4528fd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4086295e),segFN=segments_1,generation=1}
   [junit4]   2> 655456 T2666 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 655463 T2666 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 655464 T2666 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 655464 T2666 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 655464 T2666 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 655465 T2666 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 655465 T2666 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 655465 T2666 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 655466 T2666 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 655466 T2666 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 655486 T2666 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 655488 T2666 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 655491 T2666 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 655493 T2666 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 655502 T2666 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/segments,/schema,/admin/properties,/update/csv,/admin/file,/admin/mbeans,/config,/admin/luke,/admin/plugins,/admin/ping,/admin/threads,/update/json,/update/json/docs,/update,/get,/admin/system,/replication,standard,/admin/logging
   [junit4]   2> 655504 T2666 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 655506 T2666 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 655506 T2666 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 655508 T2666 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 655508 T2666 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 655509 T2666 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=46, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.15818545914049942]
   [junit4]   2> 655510 T2666 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5c4528fd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4086295e),segFN=segments_1,generation=1}
   [junit4]   2> 655510 T2666 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 655511 T2666 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@1e20960f[collection1] main
   [junit4]   2> 655511 T2666 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 655512 T2666 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 655512 T2666 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 655513 T2666 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 655513 T2666 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 655513 T2666 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 655513 T2666 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 655514 T2666 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 655514 T2666 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 655514 T2666 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 655515 T2667 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1e20960f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 655516 T2666 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 655517 T2670 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:32047/esw/zr collection:collection1 shard:shard1
   [junit4]   2> 655517 T2610 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 655517 T2610 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 655517 T2670 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 655521 T2670 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 655522 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 655522 T2670 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 655522 T2670 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C1809 name=collection1 org.apache.solr.core.SolrCore@1f1f534 url=http://127.0.0.1:32047/esw/zr/collection1 node=127.0.0.1:32047_esw%2Fzr C1809_STATE=coll:collection1 core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:32047/esw/zr, node_name=127.0.0.1:32047_esw%2Fzr}
   [junit4]   2> 655523 T2670 C:collection1 S:shard1 C1809 P32047 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:32047/esw/zr/collection1/
   [junit4]   2> 655523 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 655523 T2670 C:collection1 S:shard1 C1809 P32047 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 655523 T2670 C:collection1 S:shard1 C1809 P32047 oasc.SyncStrategy.syncToMe http://127.0.0.1:32047/esw/zr/collection1/ has no replicas
   [junit4]   2> 655524 T2670 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:32047/esw/zr/collection1/ shard1
   [junit4]   2> 655524 T2670 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 655524 T2637 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 655525 T2665 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 655525 T2665 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 655527 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32047/esw/zr",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 655528 T2637 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 655528 T2665 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 655529 T2665 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 655530 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 655577 T2670 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:32047/esw/zr/collection1/ and leader is http://127.0.0.1:32047/esw/zr/collection1/
   [junit4]   2> 655578 T2670 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:32047/esw/zr
   [junit4]   2> 655578 T2670 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 655578 T2670 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 655578 T2670 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 655579 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 655580 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32047/esw/zr",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:32047_esw%2Fzr"} current state version: 5
   [junit4]   2> 655580 T2637 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32047/esw/zr",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:32047_esw%2Fzr"}
   [junit4]   2> 655581 T2670 C:collection1 S:shard1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 655581 T2637 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 3
   [junit4]   2> 655582 T2665 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 655582 T2665 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 656725 T2610 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores/collection1
   [junit4]   2> 656728 T2610 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001
   [junit4]   2> 656728 T2610 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 656731 T2610 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:32051
   [junit4]   2> 656731 T2610 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/esw/zr, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/tempDir-001/jetty2, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores, hostPort=32051, solrconfig=solrconfig.xml}
   [junit4]   2> 656732 T2610 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 656732 T2610 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/'
   [junit4]   2> 656762 T2610 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/solr.xml
   [junit4]   2> 656797 T2610 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores
   [junit4]   2> 656797 T2610 oasc.CoreContainer.<init> New CoreContainer 192842349
   [junit4]   2> 656797 T2610 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/]
   [junit4]   2> 656798 T2610 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/lib
   [junit4]   2> 656798 T2610 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/lib).
   [junit4]   2> 656807 T2610 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 656808 T2610 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 656808 T2610 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 656808 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 656808 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 656808 T2610 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 656809 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 656809 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 656809 T2610 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 656809 T2610 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 656809 T2610 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 656814 T2610 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 656815 T2610 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 656815 T2610 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 656815 T2610 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 656815 T2610 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:32033/solr
   [junit4]   2> 656816 T2610 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 656816 T2610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 656817 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 656818 T2682 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@60b18fb0 name:ZooKeeperConnection Watcher:127.0.0.1:32033 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 656819 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 656819 T2610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 656821 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 656822 T2685 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@21576f30 name:ZooKeeperConnection Watcher:127.0.0.1:32033/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 656822 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 656825 T2610 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 657829 T2610 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32051_esw%2Fzr
   [junit4]   2> 657830 T2610 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:32051_esw%2Fzr
   [junit4]   2> 657833 T2610 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 657835 T2610 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores
   [junit4]   2> 657838 T2610 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, dataDir=data/, shard=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores/collection1/, loadOnStartup=true, collection=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores/collection1, name=collection1, transient=false, schema=schema.xml, coreNodeName=}
   [junit4]   2> 657838 T2610 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores/collection1/
   [junit4]   2> 657838 T2610 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 657840 T2686 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 657840 T2686 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 657841 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 657842 T2686 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 657842 T2686 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 657842 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32051/esw/zr",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:32051_esw%2Fzr"} current state version: 5
   [junit4]   2> 657843 T2686 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 657843 T2637 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32051/esw/zr",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:32051_esw%2Fzr"}
   [junit4]   2> 657843 T2637 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 657843 T2637 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 657843 T2686 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 657844 T2686 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 657844 T2637 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 4
   [junit4]   2> 657845 T2685 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 3)
   [junit4]   2> 657845 T2665 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 3)
   [junit4]   2> 657845 T2685 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 5 
   [junit4]   2> 657845 T2665 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 5 
   [junit4]   2> 658844 T2686 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 658845 T2686 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 658845 T2686 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 658846 T2686 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 658846 T2686 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 658846 T2686 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores/collection1/'
   [junit4]   2> 658865 T2686 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 658867 T2686 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 658877 T2686 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 658899 T2686 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 658948 T2686 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 658949 T2686 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 658959 T2686 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 659306 T2686 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 659309 T2686 C:collection

[...truncated too long message...]

ds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=492,cumulative_deletesById=226,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=56680,transaction_logs_total_number=1}
   [junit4]   2> 819687 T2729 C:collection1 S:shard1 c:collection1 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 819687 T2729 C:collection1 S:shard1 c:collection1 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 819687 T2729 C:collection1 S:shard1 c:collection1 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 819689 T2729 C:collection1 S:shard1 c:collection1 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 819697 T2729 C:collection1 S:shard1 c:collection1 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 819697 T2729 C:collection1 S:shard1 c:collection1 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores/collection1/data;done=false>>]
   [junit4]   2> 819697 T2729 C:collection1 S:shard1 c:collection1 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores/collection1/data
   [junit4]   2> 819698 T2729 C:collection1 S:shard1 c:collection1 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores/collection1/data/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores/collection1/data/index;done=false>>]
   [junit4]   2> 819698 T2729 C:collection1 S:shard1 c:collection1 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores/collection1/data/index
   [junit4]   2> 820389 T2736 C:collection1 S:shard1 c:collection1 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 820391 T2610 C:collection1 S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/esw/zr,null}
   [junit4]   2> 820594 T2610 C:collection1 S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:32033 32033
   [junit4]   2> 820765 T2611 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:32033 32033
   [junit4]   2> 820767 T2611 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		5	/solr/aliases.json
   [junit4]   2> 		5	/solr/clusterstate.json
   [junit4]   2> 		4	/solr/configs/conf1
   [junit4]   2> 		3	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent data watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		2	/solr/overseer_elect/election/93624945334288391-127.0.0.1:32047_esw%2Fzr-n_0000000001
   [junit4]   2> 		2	/solr/collections/collection1/leader_elect/shard1/election/93624945334288391-core_node1-n_0000000000
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryZkTest -Dtests.method=test -Dtests.seed=CED94C4AA2172441 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ja -Dtests.timezone=Canada/Saskatchewan -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE  173s J3 | RecoveryZkTest.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: shard1 is not consistent.  Got 963 from http://127.0.0.1:32047/esw/zr/collection1lastClient and got 272 from http://127.0.0.1:32051/esw/zr/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([CED94C4AA2172441:468D73900CEB49B9]:0)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:123)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 820776 T2610 C:collection1 S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001
   [junit4]   2> 172645 T2609 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {a_t=PostingsFormat(name=LuceneVarGapFixedInterval), a_i=PostingsFormat(name=LuceneVarGapDocFreqInterval), id=PostingsFormat(name=LuceneVarGapDocFreqInterval), rnd_b=PostingsFormat(name=LuceneVarGapDocFreqInterval), _version_=PostingsFormat(name=LuceneVarGapFixedInterval)}, docValues:{}, sim=DefaultSimilarity, locale=ja, timezone=Canada/Saskatchewan
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=75957512,total=310378496
   [junit4]   2> NOTE: All tests run in this JVM: [DeleteInactiveReplicaTest, TestFuzzyAnalyzedSuggestions, DeleteShardTest, TestRebalanceLeaders, SchemaVersionSpecificBehaviorTest, SuggesterTest, ConnectionManagerTest, DistribJoinFromCollectionTest, TestCloudSchemaless, TestPivotHelperCode, SortByFunctionTest, UUIDFieldTest, TestReqParamsAPI, TestSimpleTrackingShardHandler, TestSearchPerf, TestJoin, TestUpdate, CurrencyFieldOpenExchangeTest, BasicZkTest, DebugComponentTest, TestHashPartitioner, TestWordDelimiterFilterFactory, OutOfBoxZkACLAndCredentialsProvidersTest, UUIDUpdateProcessorFallbackTest, DirectUpdateHandlerOptimizeTest, ZkNodePropsTest, TestJsonRequest, NumericFieldsTest, TestSolrConfigHandlerConcurrent, HdfsSyncSliceTest, ActionThrottleTest, CacheHeaderTest, TestFieldTypeCollectionResource, TestRequestStatusCollectionAPI, TestBinaryField, TestNonDefinedSimilarityFactory, TestCharFilters, PingRequestHandlerTest, InfoHandlerTest, DirectSolrConnectionTest, ScriptEngineTest, ClusterStateUpdateTest, TestMergePolicyConfig, TestQueryTypes, HighlighterTest, DateRangeFieldTest, TestDefaultSearchFieldResource, TestClassNameShortening, TestJettySolrRunner, RecoveryZkTest]
   [junit4] Completed [153/481] on J3 in 173.44s, 1 test, 1 failure <<< FAILURES!

[...truncated 1047 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:484: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:229: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:510: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1434: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:991: There were test failures: 481 suites, 1911 tests, 1 failure, 62 ignored (22 assumptions)

Total time: 57 minutes 57 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 230896502 bytes
Compression is 0.0%
Took 42 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



Re: [JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2917 - Still Failing

Posted by Steve Rowe <sa...@gmail.com>.
I logged into the lucene slave and ran ant clean-jars as jenkins on each of the non-4.x workspaces containing *-1.7.6.jar, so we shouldn’t see this particular problem again - here’s the list of affected workspaces:

Lucene-Solr-Tests-trunk-Java7  (obsolete)
Lucene-Solr-SmokeRelease-trunk (disabled)
Lucene-Solr-Tests-5.x-Java7
Lucene-Solr-Tests-trunk-Java8  (disabled)
Lucene-Solr-Clover-trunk       (disabled)
Lucene-Solr-NightlyTests-trunk (disabled)

Steve

> On Apr 9, 2015, at 9:53 AM, Apache Jenkins Server <je...@builds.apache.org> wrote:
> 
> Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2917/
> 
> All tests passed
> 
> Build Log:
> [...truncated 28458 lines...]
> check-licenses:
>     [echo] License check under: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr
> [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/jcl-over-slf4j-1.7.6.jar
> [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/jcl-over-slf4j-1.7.6.jar.sha1
> 
> [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/jul-to-slf4j-1.7.6.jar
> [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/jul-to-slf4j-1.7.6.jar.sha1
> [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/slf4j-api-1.7.6.jar
> [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/slf4j-api-1.7.6.jar.sha1
> 
> [...truncated 3 lines...]
> BUILD FAILED
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:536: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:97: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:294: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/tools/custom-tasks.xml:62: License check failed. Check the logs.
> If you recently modified ivy-versions.properties or any module's ivy.xml,
> make sure you run "ant clean-jars jar-checksums" before running precommit.
> 
> Total time: 90 minutes 29 seconds
> Build step 'Invoke Ant' marked build as failure
> Archiving artifacts
> Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
> Archived 1 artifacts
> Archive block size is 32768
> Received 0 blocks and 464 bytes
> Compression is 0.0%
> Took 0.18 sec
> Recording test results
> Email was triggered for: Failure
> Sending email for trigger: Failure
> 
> 
> 
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2917 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2917/

All tests passed

Build Log:
[...truncated 28458 lines...]
check-licenses:
     [echo] License check under: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr
 [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/jcl-over-slf4j-1.7.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/jcl-over-slf4j-1.7.6.jar.sha1

 [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/jul-to-slf4j-1.7.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/jul-to-slf4j-1.7.6.jar.sha1
 [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/slf4j-api-1.7.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/slf4j-api-1.7.6.jar.sha1

[...truncated 3 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:97: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:294: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/tools/custom-tasks.xml:62: License check failed. Check the logs.
If you recently modified ivy-versions.properties or any module's ivy.xml,
make sure you run "ant clean-jars jar-checksums" before running precommit.

Total time: 90 minutes 29 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 1 artifacts
Archive block size is 32768
Received 0 blocks and 464 bytes
Compression is 0.0%
Took 0.18 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2916 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2916/

1 tests failed.
REGRESSION:  org.apache.solr.schema.TestCloudSchemaless.test

Error Message:
QUERY FAILED: xpath=/response/arr[@name='fields']/lst/str[@name='name'][.='newTestFieldInt448']  request=/schema/fields?wt=xml  response=<?xml version="1.0" encoding="UTF-8"?> <response>  <lst name="responseHeader">   <int name="status">0</int>   <int name="QTime">1873</int> </lst> <arr name="fields">   <lst>     <str name="name">_version_</str>     <str name="type">long</str>     <bool name="stored">true</bool>     <bool name="indexed">true</bool>   </lst>   <lst>     <str name="name">constantField</str>     <str name="type">tdouble</str>   </lst>   <lst>     <str name="name">id</str>     <str name="type">string</str>     <bool name="stored">true</bool>     <bool name="indexed">true</bool>     <bool name="required">true</bool>     <bool name="multiValued">false</bool>     <bool name="uniqueKey">true</bool>   </lst>   <lst>     <str name="name">newTestFieldInt0</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt1</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt10</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt100</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt101</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt102</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt103</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt104</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt105</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt106</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt107</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt108</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt109</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt11</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt110</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt111</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt112</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt113</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt114</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt115</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt116</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt117</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt118</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt119</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt12</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt120</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt121</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt122</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt123</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt124</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt125</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt126</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt127</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt128</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt129</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt13</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt130</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt131</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt132</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt133</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt134</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt135</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt136</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt137</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt138</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt139</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt14</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt140</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt141</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt142</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt143</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt144</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt145</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt146</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt147</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt148</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt149</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt15</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt150</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt151</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt152</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt153</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt154</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt155</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt156</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt157</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt158</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt159</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt16</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt160</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt161</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt162</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt163</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt164</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt165</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt166</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt167</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt168</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt169</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt17</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt170</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt171</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt172</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt173</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt174</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt175</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt176</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt177</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt178</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt179</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt18</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt180</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt181</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt182</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt183</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt184</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt185</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt186</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt187</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt188</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt189</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt19</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt190</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt191</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt192</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt193</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt194</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt195</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt196</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt197</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt198</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt199</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt2</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt20</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt200</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt201</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt202</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt203</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt204</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt205</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt206</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt207</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt208</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt209</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt21</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt210</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt211</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt212</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt213</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt214</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt215</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt216</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt217</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt218</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt219</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt22</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt220</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt221</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt222</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt223</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt224</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt225</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt226</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt227</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt228</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt229</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt23</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt230</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt231</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt232</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt233</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt234</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt235</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt236</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt237</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt238</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt239</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt24</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt240</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt241</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt242</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt243</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt244</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt245</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt246</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt247</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt248</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt249</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt25</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt250</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt251</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt252</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt253</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt254</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt255</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt256</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt257</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt258</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt259</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt26</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt260</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt261</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt262</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt263</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt264</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt265</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt266</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt267</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt268</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt269</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt27</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt270</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt271</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt272</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt273</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt274</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt275</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt276</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt277</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt278</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt279</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt28</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt280</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt281</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt282</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt283</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt284</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt285</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt286</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt287</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt288</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt289</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt29</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt290</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt291</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt292</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt293</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt294</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt295</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt296</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt297</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt298</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt299</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt3</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt30</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt300</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt301</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt302</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt303</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt304</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt305</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt306</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt307</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt308</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt309</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt31</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt310</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt311</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt312</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt313</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt314</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt315</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt316</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt317</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt318</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt319</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt32</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt320</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt321</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt322</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt323</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt324</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt325</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt326</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt327</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt328</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt329</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt33</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt330</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt331</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt332</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt333</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt334</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt335</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt336</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt337</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt338</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt339</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt34</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt340</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt341</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt342</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt343</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt344</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt345</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt346</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt347</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt348</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt349</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt35</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt350</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt351</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt352</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt353</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt354</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt355</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt356</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt357</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt358</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt359</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt36</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt360</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt361</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt362</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt363</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt364</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt365</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt366</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt367</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt368</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt369</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt37</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt370</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt371</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt372</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt373</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt374</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt375</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt376</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt377</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt378</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt379</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt38</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt380</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt381</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt382</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt383</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt384</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt385</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt386</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt387</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt388</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt389</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt39</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt390</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt391</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt392</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt393</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt394</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt395</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt396</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt397</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt398</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt399</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt4</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt40</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt400</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt401</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt402</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt403</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt404</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt405</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt406</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt407</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt408</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt409</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt41</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt410</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt411</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt412</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt413</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt414</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt415</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt416</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt417</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt418</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt419</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt42</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt420</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt421</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt422</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt423</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt424</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt425</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt426</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt427</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt428</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt429</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt43</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt430</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt431</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt432</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt433</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt434</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt435</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt436</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt437</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt438</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt439</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt44</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt440</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt441</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt442</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt443</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt444</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt445</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt446</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt447</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt449</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt45</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt46</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt47</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt48</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt49</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt5</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt50</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt51</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt52</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt53</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt54</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt55</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt56</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt57</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt58</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt59</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt6</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt60</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt61</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt62</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt63</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt64</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt65</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt66</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt67</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt68</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt69</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt7</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt70</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt71</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt72</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt73</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt74</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt75</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt76</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt77</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt78</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt79</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt8</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt80</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt81</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt82</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt83</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt84</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt85</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt86</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt87</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt88</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt89</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt9</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt90</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt91</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt92</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt93</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt94</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt95</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt96</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt97</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt98</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt99</str>     <str name="type">tlong</str>   </lst> </arr> </response> 

Stack Trace:
java.lang.AssertionError: QUERY FAILED: xpath=/response/arr[@name='fields']/lst/str[@name='name'][.='newTestFieldInt448']  request=/schema/fields?wt=xml  response=<?xml version="1.0" encoding="UTF-8"?>
<response>

<lst name="responseHeader">
  <int name="status">0</int>
  <int name="QTime">1873</int>
</lst>
<arr name="fields">
  <lst>
    <str name="name">_version_</str>
    <str name="type">long</str>
    <bool name="stored">true</bool>
    <bool name="indexed">true</bool>
  </lst>
  <lst>
    <str name="name">constantField</str>
    <str name="type">tdouble</str>
  </lst>
  <lst>
    <str name="name">id</str>
    <str name="type">string</str>
    <bool name="stored">true</bool>
    <bool name="indexed">true</bool>
    <bool name="required">true</bool>
    <bool name="multiValued">false</bool>
    <bool name="uniqueKey">true</bool>
  </lst>
  <lst>
    <str name="name">newTestFieldInt0</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt1</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt10</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt100</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt101</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt102</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt103</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt104</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt105</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt106</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt107</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt108</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt109</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt11</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt110</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt111</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt112</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt113</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt114</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt115</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt116</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt117</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt118</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt119</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt12</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt120</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt121</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt122</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt123</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt124</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt125</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt126</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt127</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt128</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt129</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt13</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt130</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt131</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt132</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt133</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt134</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt135</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt136</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt137</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt138</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt139</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt14</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt140</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt141</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt142</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt143</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt144</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt145</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt146</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt147</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt148</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt149</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt15</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt150</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt151</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt152</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt153</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt154</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt155</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt156</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt157</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt158</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt159</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt16</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt160</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt161</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt162</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt163</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt164</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt165</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt166</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt167</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt168</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt169</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt17</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt170</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt171</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt172</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt173</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt174</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt175</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt176</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt177</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt178</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt179</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt18</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt180</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt181</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt182</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt183</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt184</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt185</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt186</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt187</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt188</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt189</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt19</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt190</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt191</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt192</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt193</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt194</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt195</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt196</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt197</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt198</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt199</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt2</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt20</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt200</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt201</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt202</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt203</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt204</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt205</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt206</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt207</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt208</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt209</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt21</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt210</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt211</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt212</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt213</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt214</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt215</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt216</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt217</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt218</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt219</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt22</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt220</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt221</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt222</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt223</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt224</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt225</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt226</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt227</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt228</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt229</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt23</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt230</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt231</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt232</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt233</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt234</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt235</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt236</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt237</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt238</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt239</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt24</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt240</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt241</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt242</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt243</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt244</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt245</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt246</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt247</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt248</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt249</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt25</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt250</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt251</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt252</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt253</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt254</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt255</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt256</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt257</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt258</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt259</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt26</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt260</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt261</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt262</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt263</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt264</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt265</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt266</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt267</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt268</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt269</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt27</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt270</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt271</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt272</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt273</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt274</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt275</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt276</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt277</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt278</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt279</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt28</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt280</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt281</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt282</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt283</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt284</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt285</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt286</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt287</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt288</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt289</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt29</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt290</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt291</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt292</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt293</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt294</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt295</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt296</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt297</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt298</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt299</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt3</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt30</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt300</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt301</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt302</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt303</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt304</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt305</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt306</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt307</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt308</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt309</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt31</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt310</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt311</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt312</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt313</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt314</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt315</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt316</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt317</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt318</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt319</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt32</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt320</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt321</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt322</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt323</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt324</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt325</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt326</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt327</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt328</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt329</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt33</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt330</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt331</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt332</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt333</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt334</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt335</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt336</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt337</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt338</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt339</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt34</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt340</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt341</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt342</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt343</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt344</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt345</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt346</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt347</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt348</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt349</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt35</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt350</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt351</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt352</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt353</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt354</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt355</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt356</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt357</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt358</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt359</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt36</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt360</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt361</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt362</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt363</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt364</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt365</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt366</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt367</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt368</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt369</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt37</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt370</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt371</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt372</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt373</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt374</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt375</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt376</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt377</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt378</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt379</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt38</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt380</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt381</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt382</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt383</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt384</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt385</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt386</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt387</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt388</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt389</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt39</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt390</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt391</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt392</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt393</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt394</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt395</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt396</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt397</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt398</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt399</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt4</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt40</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt400</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt401</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt402</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt403</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt404</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt405</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt406</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt407</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt408</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt409</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt41</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt410</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt411</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt412</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt413</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt414</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt415</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt416</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt417</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt418</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt419</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt42</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt420</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt421</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt422</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt423</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt424</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt425</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt426</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt427</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt428</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt429</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt43</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt430</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt431</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt432</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt433</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt434</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt435</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt436</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt437</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt438</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt439</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt44</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt440</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt441</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt442</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt443</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt444</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt445</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt446</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt447</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt449</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt45</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt46</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt47</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt48</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt49</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt5</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt50</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt51</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt52</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt53</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt54</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt55</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt56</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt57</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt58</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt59</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt6</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt60</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt61</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt62</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt63</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt64</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt65</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt66</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt67</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt68</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt69</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt7</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt70</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt71</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt72</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt73</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt74</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt75</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt76</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt77</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt78</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt79</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt8</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt80</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt81</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt82</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt83</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt84</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt85</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt86</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt87</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt88</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt89</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt9</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt90</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt91</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt92</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt93</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt94</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt95</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt96</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt97</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt98</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt99</str>
    <str name="type">tlong</str>
  </lst>
</arr>
</response>

	at __randomizedtesting.SeedInfo.seed([C8B1BCB8C8FBCD32:40E583626607A0CA]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.schema.TestCloudSchemaless.test(TestCloudSchemaless.java:159)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakCon

[...truncated too long message...]

4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt68</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt69</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt7</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt70</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt71</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt72</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt73</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt74</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt75</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt76</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt77</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt78</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt79</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt8</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt80</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt81</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt82</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt83</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt84</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt85</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt86</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt87</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt88</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt89</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt9</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt90</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt91</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt92</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt93</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt94</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt95</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt96</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt97</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt98</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt99</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    > </arr>
   [junit4]    > </response>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([C8B1BCB8C8FBCD32:40E583626607A0CA]:0)
   [junit4]    > 	at org.apache.solr.schema.TestCloudSchemaless.test(TestCloudSchemaless.java:159)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1371084 T3547 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.schema.TestCloudSchemaless C8B1BCB8C8FBCD32-001
   [junit4]   2> 101046 T3546 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=true,coord=no): {}, locale=et, timezone=Pacific/Honolulu
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=142490696,total=348651520
   [junit4]   2> NOTE: All tests run in this JVM: [TestCoreDiscovery, TestBlobHandler, TestLuceneMatchVersion, TestDynamicLoading, TestDistributedSearch, ExternalCollectionsTest, TestDFRSimilarityFactory, DistributedQueryElevationComponentTest, TestSearcherReuse, ConnectionManagerTest, QueryResultKeyTest, DistribCursorPagingTest, TestBlendedInfixSuggestions, OpenCloseCoreStressTest, TestCloudPivotFacet, SolrCmdDistributorTest, DefaultValueUpdateProcessorTest, TestNonDefinedSimilarityFactory, PluginInfoTest, TestArbitraryIndexDir, TestPerFieldSimilarity, PrimitiveFieldTypeTest, TestRequestStatusCollectionAPI, TestTrie, LeaderInitiatedRecoveryOnCommitTest, HdfsChaosMonkeySafeLeaderTest, BasicFunctionalityTest, ResponseLogComponentTest, TestChildDocTransformer, TestStressUserVersions, TestReplicaProperties, TestImplicitCoreProperties, TestSurroundQueryParser, DistributedSuggestComponentTest, TestSolrQueryParser, SolrXmlInZkTest, BadIndexSchemaTest, TestBadConfig, IndexSchemaRuntimeFieldTest, TestJmxIntegration, TestReload, SpellCheckComponentTest, TestConfigSets, SyncSliceTest, TestRealTimeGet, ChangedSchemaMergeTest, CacheHeaderTest, TestCloudSchemaless]
   [junit4] Completed on J2 in 101.36s, 1 test, 1 failure <<< FAILURES!

[...truncated 763 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 481 suites, 1911 tests, 1 failure, 62 ignored (22 assumptions)

Total time: 57 minutes 51 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 206525730 bytes
Compression is 0.0%
Took 39 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2915 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2915/

1 tests failed.
REGRESSION:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
Didn't see all replicas for shard shard1 in collection1 come up within 30000 ms! ClusterState: {   "collection1":{     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{           "core_node1":{             "core":"collection1",             "state":"active",             "base_url":"http://127.0.0.1:30988/h/kg",             "node_name":"127.0.0.1:30988_h%2Fkg",             "leader":"true"},           "core_node2":{             "core":"collection1",             "state":"recovering",             "base_url":"http://127.0.0.1:45044/h/kg",             "node_name":"127.0.0.1:45044_h%2Fkg"}}}},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "replicationFactor":"1",     "autoCreated":"true",     "router":{"name":"compositeId"}},   "control_collection":{     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{"core_node1":{             "core":"collection1",             "state":"active",             "base_url":"http://127.0.0.1:57657/h/kg",             "node_name":"127.0.0.1:57657_h%2Fkg",             "leader":"true"}}}},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "replicationFactor":"1",     "autoCreated":"true",     "router":{"name":"compositeId"}}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collection1 come up within 30000 ms! ClusterState: {
  "collection1":{
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node1":{
            "core":"collection1",
            "state":"active",
            "base_url":"http://127.0.0.1:30988/h/kg",
            "node_name":"127.0.0.1:30988_h%2Fkg",
            "leader":"true"},
          "core_node2":{
            "core":"collection1",
            "state":"recovering",
            "base_url":"http://127.0.0.1:45044/h/kg",
            "node_name":"127.0.0.1:45044_h%2Fkg"}}}},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "replicationFactor":"1",
    "autoCreated":"true",
    "router":{"name":"compositeId"}},
  "control_collection":{
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node1":{
            "core":"collection1",
            "state":"active",
            "base_url":"http://127.0.0.1:57657/h/kg",
            "node_name":"127.0.0.1:57657_h%2Fkg",
            "leader":"true"}}}},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "replicationFactor":"1",
    "autoCreated":"true",
    "router":{"name":"compositeId"}}}
	at __randomizedtesting.SeedInfo.seed([6F561FCF85559A56:E70220152BA9F7AE]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:1920)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:102)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 9383 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/init-core-data-001
   [junit4]   2> 208180 T322 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /h/kg
   [junit4]   2> 208192 T322 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 208193 T323 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 208292 T322 oasc.ZkTestServer.run start zk server on port:48644
   [junit4]   2> 208293 T322 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 208294 T322 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 208299 T330 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5529d23e name:ZooKeeperConnection Watcher:127.0.0.1:48644 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 208300 T322 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 208301 T322 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 208302 T322 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 208306 T322 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 208308 T322 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 208310 T333 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@29f065a8 name:ZooKeeperConnection Watcher:127.0.0.1:48644/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 208310 T322 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 208311 T322 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 208312 T322 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 208316 T322 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 208318 T322 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 208320 T322 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 208325 T322 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 208326 T322 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 208330 T322 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 208331 T322 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 208334 T322 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 208336 T322 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 208339 T322 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 208340 T322 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 208343 T322 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 208344 T322 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 208346 T322 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 208347 T322 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 208350 T322 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 208351 T322 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 208353 T322 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 208354 T322 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 208356 T322 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 208357 T322 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 208360 T322 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 208361 T322 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 208364 T322 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 208364 T322 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 210925 T322 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores/collection1
   [junit4]   2> 210931 T322 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 210937 T322 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:58650
   [junit4]   2> 210937 T322 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/h/kg, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/tempDir-001/control/data, hostPort=57657}
   [junit4]   2> 210938 T322 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 210939 T322 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/'
   [junit4]   2> 211003 T322 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/solr.xml
   [junit4]   2> 211082 T322 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores
   [junit4]   2> 211082 T322 oasc.CoreContainer.<init> New CoreContainer 847984524
   [junit4]   2> 211083 T322 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/]
   [junit4]   2> 211083 T322 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/lib
   [junit4]   2> 211085 T322 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/lib).
   [junit4]   2> 211101 T322 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 211101 T322 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 211102 T322 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 211102 T322 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 211103 T322 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 211103 T322 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 211104 T322 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 211104 T322 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 211104 T322 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 211105 T322 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 211106 T322 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 211116 T322 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 211118 T322 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 211119 T322 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 211119 T322 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 211119 T322 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48644/solr
   [junit4]   2> 211120 T322 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 211121 T322 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 211122 T322 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 211124 T345 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7b33f19f name:ZooKeeperConnection Watcher:127.0.0.1:48644 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 211125 T322 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 211126 T322 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 211129 T322 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 211131 T348 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@38433773 name:ZooKeeperConnection Watcher:127.0.0.1:48644/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 211132 T322 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 211134 T322 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 211136 T322 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 211139 T322 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 211141 T322 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 211143 T322 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 211145 T322 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 211147 T322 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 211149 T322 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 211150 T322 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57657_h%2Fkg
   [junit4]   2> 211151 T322 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:57657_h%2Fkg
   [junit4]   2> 211153 T322 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 211155 T322 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 211156 T322 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 211158 T322 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:57657_h%2Fkg
   [junit4]   2> 211158 T322 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 211160 T322 oasc.Overseer.start Overseer (id=93621668344889347-127.0.0.1:57657_h%2Fkg-n_0000000000) starting
   [junit4]   2> 211163 T322 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 211169 T322 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 211170 T350 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 211170 T349 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 211170 T322 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 211175 T322 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores
   [junit4]   2> 211183 T322 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores/collection1, shard=, coreNodeName=, name=collection1, dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores/collection1/, config=solrconfig.xml, collection=control_collection, transient=false, loadOnStartup=true}
   [junit4]   2> 211183 T322 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores/collection1/
   [junit4]   2> 211184 T322 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 211185 T352 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 211185 T352 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 211187 T352 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 211187 T348 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 211188 T349 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57657/h/kg",
   [junit4]   2> 	  "node_name":"127.0.0.1:57657_h%2Fkg",
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 211188 T349 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57657/h/kg",
   [junit4]   2> 	  "node_name":"127.0.0.1:57657_h%2Fkg",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 211189 T349 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 211189 T349 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 211190 T348 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 212187 T352 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 212188 T352 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 212188 T352 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 212189 T352 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 212189 T352 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 212190 T352 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores/collection1/'
   [junit4]   2> 212205 T352 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 212207 T352 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 212218 T352 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 212246 T352 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 212334 T352 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 212336 T352 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 212352 T352 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 212772 T352 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 212793 T352 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 212796 T352 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 212812 T352 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 212816 T352 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 212819 T352 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 212820 T352 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 212820 T352 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 212820 T352 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 212821 T352 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 212821 T352 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 212821 T352 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 212822 T352 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 212822 T352 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores/collection1/data/
   [junit4]   2> 212823 T352 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@62f98b99
   [junit4]   2> 212823 T352 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores/collection1/data
   [junit4]   2> 212824 T352 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores/collection1/data/index/
   [junit4]   2> 212824 T352 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 212824 T352 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores/collection1/data/index
   [junit4]   2> 212825 T352 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=42, maxMergedSegmentMB=88.0810546875, floorSegmentMB=1.7578125, forceMergeDeletesPctAllowed=4.679093429411079, segmentsPerTier=15.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 212826 T352 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@348d830c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3696a3c8),segFN=segments_1,generation=1}
   [junit4]   2> 212826 T352 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 212840 T352 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 212840 T352 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 212841 T352 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 212841 T352 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 212841 T352 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 212842 T352 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 212842 T352 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 212842 T352 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 212843 T352 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 212855 T352 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 212857 T352 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 212858 T352 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 212860 T352 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 212868 T352 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/system,/admin/segments,/get,/update/json/docs,/update/csv,/admin/ping,/admin/file,/schema,/admin/properties,standard,/admin/luke,/config,/admin/mbeans,/update,/admin/logging,/admin/plugins,/update/json,/admin/threads,/replication
   [junit4]   2> 212874 T352 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 212876 T352 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 212876 T352 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 212877 T352 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 212877 T352 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 212878 T352 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=25, maxMergedSegmentMB=15.447265625, floorSegmentMB=1.6923828125, forceMergeDeletesPctAllowed=1.7328479877713976, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 212879 T352 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@348d830c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3696a3c8),segFN=segments_1,generation=1}
   [junit4]   2> 212879 T352 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 212880 T352 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@41dbc868[collection1] main
   [junit4]   2> 212881 T352 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 212881 T352 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 212881 T352 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 212882 T352 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 212882 T352 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 212882 T352 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 212883 T352 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 212883 T352 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 212883 T352 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 212884 T352 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 212885 T353 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@41dbc868[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 212885 T352 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 212886 T356 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:57657/h/kg collection:control_collection shard:shard1
   [junit4]   2> 212886 T322 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 212887 T322 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 212887 T356 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 212889 T322 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 212890 T322 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 212891 T360 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3b182ea8 name:ZooKeeperConnection Watcher:127.0.0.1:48644/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 212891 T356 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 212891 T322 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 212892 T322 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 212893 T322 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 212893 T348 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 212893 T356 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 212894 T356 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 212894 T349 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C27 name=collection1 org.apache.solr.core.SolrCore@289cee0a url=http://127.0.0.1:57657/h/kg/collection1 node=127.0.0.1:57657_h%2Fkg C27_STATE=coll:control_collection core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:57657/h/kg, node_name=127.0.0.1:57657_h%2Fkg}
   [junit4]   2> 212894 T356 C:control_collection S:shard1 C27 P57657 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:57657/h/kg/collection1/
   [junit4]   2> 212895 T356 C:control_collection S:shard1 C27 P57657 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 212895 T356 C:control_collection S:shard1 C27 P57657 oasc.SyncStrategy.syncToMe http://127.0.0.1:57657/h/kg/collection1/ has no replicas
   [junit4]   2> 212895 T356 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:57657/h/kg/collection1/ shard1
   [junit4]   2> 212895 T356 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 212895 T360 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 212895 T322 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 212895 T348 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 212899 T348 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 212900 T349 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57657/h/kg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 213003 T360 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 213003 T348 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 213051 T356 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:57657/h/kg/collection1/ and leader is http://127.0.0.1:57657/h/kg/collection1/
   [junit4]   2> 213052 T356 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:57657/h/kg
   [junit4]   2> 213052 T356 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 213053 T356 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 213053 T356 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 213055 T348 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 213056 T349 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57657/h/kg",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:57657_h%2Fkg",
   [junit4]   2> 	  "operation":"state"} current state version: 3
   [junit4]   2> 213056 T349 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57657/h/kg",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:57657_h%2Fkg",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 213160 T348 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 213160 T360 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 214571 T322 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores/collection1
   [junit4]   2> 214574 T322 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001
   [junit4]   2> 214574 T322 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 214577 T322 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:33918
   [junit4]   2> 214577 T322 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solrconfig=solrconfig.xml, hostPort=30988, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/tempDir-001/jetty1, hostContext=/h/kg, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores}
   [junit4]   2> 214578 T322 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 214578 T322 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/'
   [junit4]   2> 214650 T322 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/solr.xml
   [junit4]   2> 214715 T322 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores
   [junit4]   2> 214715 T322 oasc.CoreContainer.<init> New CoreContainer 1852987774
   [junit4]   2> 214715 T322 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/]
   [junit4]   2> 214716 T322 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/lib
   [junit4]   2> 214719 T322 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/lib).
   [junit4]   2> 214733 T322 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 214733 T322 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 214734 T322 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 214734 T322 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 214734 T322 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 214735 T322 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 214735 T322 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 214735 T322 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 214736 T322 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 214736 T322 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 214736 T322 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 214741 T322 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 214748 T322 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 214748 T322 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 214749 T322 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 214749 T322 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48644/solr
   [junit4]   2> 214749 T322 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 214750 T322 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 214750 T322 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 214752 T372 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@670e068d name:ZooKeeperConnection Watcher:127.0.0.1:48644 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 214753 T322 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 214755 T322 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 214758 T322 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 214760 T375 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@549692e9 name:ZooKeeperConnection Watcher:127.0.0.1:48644/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 214760 T322 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 214766 T322 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 215771 T322 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:30988_h%2Fkg
   [junit4]   2> 215772 T322 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:30988_h%2Fkg
   [junit4]   2> 215775 T322 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 215778 T322 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores
   [junit4]   2> 215781 T322 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, schema=schema.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores/collection1/, loadOnStartup=true, transient=false, name=collection1, shard=, collection=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores/collection1, coreNodeName=, dataDir=data/}
   [junit4]   2> 215781 T322 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores/collection1/
   [junit4]   2> 215782 T322 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 215783 T376 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 215784 T376 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 215785 T376 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 215785 T348 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 215787 T349 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30988/h/kg",
   [junit4]   2> 	  "node_name":"127.0.0.1:30988_h%2Fkg",
   [junit4]   2> 	  "operation":"state"} current state version: 4
   [junit4]   2> 215787 T349 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30988/h/kg",
   [junit4]   2> 	  "node_name":"127.0.0.1:30988_h%2Fkg",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 215788 T349 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 215788 T349 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 215790 T360 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 215790 T348 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 215790 T375 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 216786 T376 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 216787 T376 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 216788 T376 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 216788 T376 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 216789 T376 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 216789 T376 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores/collection1/'
   [junit4]   2> 216808 T376 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 216810 T376 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 216831 T376 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 216851 T376 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 216892 T376 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 216893 T376 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 216910 T376 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 217450 T376 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 217471 T376 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 217475 T376 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 217498 T376 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 217503 T376 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 217508 T376 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 217509 T376 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 217510 T376 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 217510 T376 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 217511 T376 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 217511 T376 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 217512 T376 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 217512 T376 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 217513 T376 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 217513 T376 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@62f98b99
   [junit4]   2> 217514 T376 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores/collection1/data
   [junit4]   2> 217515 T376 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 217515 T376 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 217516 T376 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 217516 T376 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=42, maxMergedSegmentMB=88.0810546875, floorSegmentMB=1.7578125, forceMergeDeletesPctAllowed=4.679093429411079, segmentsPerTier=15.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 217517 T376 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@749ce18a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@679b66d5),segFN=segments_1,generation=1}
   [junit4]   2> 217518 T376 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 217529 T376 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 217529 T376 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 217530 T376 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 217530 T376 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 217531 T376 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 217531 T376 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 217532 T376 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 217532 T376 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 217533 T376 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 217575 T376 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 217579 T376 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 217582 T376 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 217584 T376 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 217595 T376 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/system,/admin/segments,/get,/update/json/docs,/update/csv,/admin/ping,/admin/file,/schema,/admin/properties,standard,/admin/luke,/config,/admin/mbeans,/update,/admin/logging,/admin/plugins,/update/json,/admin/threads,/replication
   [junit4]   2> 217598 T376 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 217600 T376 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 217600 T376 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 217602 T376 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 217602 T376 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 217603 T376 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=25, maxMergedSegmentMB=15.447265625, floorSegmentMB=1.6923828125, forceMergeDeletesPctAllowed=1.7328479877713976, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 217604 T376 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@749ce18a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@679b66d5),segFN=segments_1,generation=1}
   [junit4]   2> 217605 T376 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 217605 T376 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@56c72790[collection1] main
   [junit4]   2> 217606 T376 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 217607 T376 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 217607 T376 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 217608 T376 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 217608 T376 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 217609 T376 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 217610 T376 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 217610 T376 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 217610 T376 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 217611 T376 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 217612 T377 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@56c72790[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 217613 T376 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 217614 T380 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:30988/h/kg collection:collection1 shard:shard1
   [junit4]   2> 217614 T322 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 217615 T322 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 217615 T380 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 217620 T380 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 217622 T348 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 217622 T380 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 217623 T380 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 217623 T349 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> ASYNC  NEW_CORE C28 name=collection1 org.apache.solr.core.SolrCore@6f2ccccc url=http://127.0.0.1:30988/h/kg/collection1 node=127.0.0.1:30988_h%2Fkg C28_STATE=coll:collection1 core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:30988/h/kg, node_name=127.0.0.1:30988_h%2Fkg}
   [junit4]   2> 217623 T380 C:collection1 S:shard1 C28 P30988 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:30988/h/kg/collection1/
   [junit4]   2> 217624 T380 C:collection1 S:shard1 C28 P30988 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 217624 T380 C:collection1 S:shard1 C28 P30988 oasc.SyncStrategy.syncToMe http://127.0.0.1:30988/h/kg/collection1/ has no replicas
   [junit4]   2> 217624 T380 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:30988/h/kg/collection1/ shard1
   [junit4]   2> 217624 T380 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 217625 T360 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 217625 T375 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 217625 T348 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 217628 T348 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 217629 T349 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30988/h/kg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 217733 T360 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 217733 T375 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 217733 T348 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 217781 T380 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:30988/h/kg/collection1/ and leader is http://127.0.0.1:30988/h/kg/collection1/
   [junit4]   2> 217782 T380 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:30988/h/kg
   [junit4]   2> 217782 T380 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 217783 T380 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 217783 T380 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 217785 T348 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 217786 T349 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30988/h/kg",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:30988_h%2Fkg",
   [junit4]   2> 	  "operation":"state"} current state version: 7
   [junit4]   2> 217787 T349 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30988/h/kg",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:30988_h%2Fkg",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 217891 T360 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 217892 T348 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 217891 T375 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 219542 T322 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores/collection1
   [junit4]   2> 219544 T322 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001
   [junit4]   2> 219545 T322 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 219548 T322 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:39770
   [junit4]   2> 219548 T322 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores, hostContext=/h/kg, hostPort=45044, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/tempDir-001/jetty2, solrconfig=solrconfig.xml}
   [junit4]   2> 219549 T322 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 219549 T322 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/'
   [junit4]   2> 219589 T322 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/solr.xml
   [junit4]   2> 219631 T322 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores
   [junit4]   2> 219632 T322 oasc.CoreContainer.<init> New CoreContainer 591304576
   [junit4]   2> 219632 T322 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/]
   [junit4]   2> 219632 T322 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/lib
   [junit4]   2> 219633 T322 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/lib).
   [junit4]   2> 219658 T322 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 219659 T322 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 219660 T322 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 219660 T322 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 219660 T322 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 219661 T322 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 219661 T322 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 219662 T322 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 219662 T322 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 219663 T322 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 219663 T322 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 219690 T322 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 219692 T322 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 219692 T322 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 219693 T322 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 219693 T322 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48644/solr
   [junit4]   2> 219694 T322 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 219694 T322 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 219695 T322 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 219697 T393 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5fb8b872 name:ZooKeeperConnection Watcher:127.0.0.1:48644 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 219698 T322 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 219699 T322 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 219701 T322 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 219703 T396 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3155d7fc name:ZooKeeperConnection Watcher:127.0.0.1:48644/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 219703 T322 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 219709 T322 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 220714 T322 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45044_h%2Fkg
   [junit4]   2> 220715 T322 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:45044_h%2Fkg
   [junit4]   2> 220718 T322 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 220721 T322 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores
   [junit4]   2> 220724 T322 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {schema=schema.xml, loadOnStartup=true, transient=false, dataDir=data/, collection=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores/collection1, coreNodeName=, shard=, config=solrconfig.xml, name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores/collection1/}
   [junit4]   2> 220725 T322 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores/collection1/
   [junit4]   2> 220726 T322 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 220727 T397 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 220728 T397 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 220729 T348 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 220729 T397 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 220730 T349 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:45044/h/kg",
   [junit4]   2> 	  "node_name":"127.0.0.1:45044_h%2Fkg",
   [junit4]   2> 	  "operation":"state"} current state version: 8
   [junit4]   2> 220730 T349 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:45044/h/kg",
   [junit4]   2> 	  "node_name":"127.0.0.1:45044_h%2Fkg",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 220731 T349 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 220731 T349 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 220732 T396 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 220732 T348 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 220732 T375 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 220732 T360 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 221730 T397 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 221731 T397 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 221732 T397 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 221732 T397 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 221732 T397 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 221733 T397 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores/collection1/'
   [junit4]   2> 221747 T397 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 221749 T397 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 221760 T397 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 221780 T397 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 221848 T397 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 221849 T397 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 221869 T397 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 222316 T397 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 222332 T397 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 222334 T397 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 222351 T397 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 222355 T397 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 222360 T397 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 222361 T397 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 222361 T397 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 222362 T397 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 222362 T397 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 222363 T397 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 222363 T397 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 222363 T397 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 222364 T397 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/sh

[...truncated too long message...]

tate.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 348207 T441 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 348211 T441 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 348240 T441 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 348240 T441 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores/collection1/data/index.20150409122150355 [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores/collection1/data/index.20150409122150355;done=false>>]
   [junit4]   2> 348241 T441 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores/collection1/data/index.20150409122150355
   [junit4]   2> 348241 T441 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores/collection1/data;done=false>>]
   [junit4]   2> 348242 T441 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores/collection1/data
   [junit4]   2> 348905 T442 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 348908 T322 C:control_collection S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/h/kg,null}
   [junit4]   2> 349022 T322 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:48644 48644
   [junit4]   2> 349180 T323 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:48644 48644
   [junit4]   2> 349181 T323 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 349182 T322 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 6 connections to: http://127.0.0.1:30988/h/kg, target: http://127.0.0.1:33918/h/kg
   [junit4]   2> 349182 T322 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:57657/h/kg, target: http://127.0.0.1:58650/h/kg
   [junit4]   2> 349182 T322 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 1 connections to: http://127.0.0.1:45044/h/kg, target: http://127.0.0.1:39770/h/kg
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=6F561FCF85559A56 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=pt_PT -Dtests.timezone=Asia/Ulaanbaatar -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE  141s J3 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Didn't see all replicas for shard shard1 in collection1 come up within 30000 ms! ClusterState: {
   [junit4]    >   "collection1":{
   [junit4]    >     "shards":{"shard1":{
   [junit4]    >         "range":"80000000-7fffffff",
   [junit4]    >         "state":"active",
   [junit4]    >         "replicas":{
   [junit4]    >           "core_node1":{
   [junit4]    >             "core":"collection1",
   [junit4]    >             "state":"active",
   [junit4]    >             "base_url":"http://127.0.0.1:30988/h/kg",
   [junit4]    >             "node_name":"127.0.0.1:30988_h%2Fkg",
   [junit4]    >             "leader":"true"},
   [junit4]    >           "core_node2":{
   [junit4]    >             "core":"collection1",
   [junit4]    >             "state":"recovering",
   [junit4]    >             "base_url":"http://127.0.0.1:45044/h/kg",
   [junit4]    >             "node_name":"127.0.0.1:45044_h%2Fkg"}}}},
   [junit4]    >     "maxShardsPerNode":"1",
   [junit4]    >     "autoAddReplicas":"false",
   [junit4]    >     "replicationFactor":"1",
   [junit4]    >     "autoCreated":"true",
   [junit4]    >     "router":{"name":"compositeId"}},
   [junit4]    >   "control_collection":{
   [junit4]    >     "shards":{"shard1":{
   [junit4]    >         "range":"80000000-7fffffff",
   [junit4]    >         "state":"active",
   [junit4]    >         "replicas":{"core_node1":{
   [junit4]    >             "core":"collection1",
   [junit4]    >             "state":"active",
   [junit4]    >             "base_url":"http://127.0.0.1:57657/h/kg",
   [junit4]    >             "node_name":"127.0.0.1:57657_h%2Fkg",
   [junit4]    >             "leader":"true"}}}},
   [junit4]    >     "maxShardsPerNode":"1",
   [junit4]    >     "autoAddReplicas":"false",
   [junit4]    >     "replicationFactor":"1",
   [junit4]    >     "autoCreated":"true",
   [junit4]    >     "router":{"name":"compositeId"}}}
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([6F561FCF85559A56:E70220152BA9F7AE]:0)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:1920)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:102)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 349231 T322 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001
   [junit4]   2> 141056 T321 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {timestamp=PostingsFormat(name=Asserting), range_facet_l=PostingsFormat(name=Asserting), a_t=PostingsFormat(name=Asserting), text=Lucene50(blocksize=128), id=PostingsFormat(name=Direct), multiDefault=PostingsFormat(name=Asserting), intDefault=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), _version_=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128)))}, docValues:{timestamp=DocValuesFormat(name=Asserting)}, sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=pt_PT, timezone=Asia/Ulaanbaatar
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=103305600,total=232783872
   [junit4]   2> NOTE: All tests run in this JVM: [TestAnalyzeInfixSuggestions, TestElisionMultitermQuery, TestCustomSort, LeaderInitiatedRecoveryOnCommitTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J3 in 141.37s, 1 test, 1 failure <<< FAILURES!

[...truncated 1372 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 480 suites, 1910 tests, 1 failure, 62 ignored (22 assumptions)

Total time: 58 minutes 50 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 211615867 bytes
Compression is 0.0%
Took 44 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2914 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2914/

1 tests failed.
REGRESSION:  org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds

Error Message:
soft529 wasn't fast enough

Stack Trace:
java.lang.AssertionError: soft529 wasn't fast enough
	at __randomizedtesting.SeedInfo.seed([BB89623F8C2FAD9B:EA5D9BBF3D5C9D3C]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNotNull(Assert.java:526)
	at org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds(SoftAutoCommitTest.java:111)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 9433 lines...]
   [junit4] Suite: org.apache.solr.update.SoftAutoCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001
   [junit4]   2> 30731 T27 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (true)
   [junit4]   2> 30733 T27 oas.SolrTestCaseJ4.initCore ####initCore
   [junit4]   2> 30734 T27 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 30735 T27 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 30736 T27 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 30737 T27 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 30909 T27 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 30986 T27 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 31119 T27 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 31121 T27 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 31215 T27 oass.IndexSchema.readSchema [null] Schema name=test
   [junit4]   2> 32831 T27 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 32884 T27 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 32891 T27 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 32943 T27 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 32949 T27 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 32954 T27 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 32960 T27 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 32960 T27 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 32961 T27 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 32962 T27 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 32962 T27 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 32991 T27 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 32991 T27 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr
   [junit4]   2> 32992 T27 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/'
   [junit4]   2> 33037 T27 oasc.CoreContainer.<init> New CoreContainer 1465666313
   [junit4]   2> 33037 T27 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/]
   [junit4]   2> 33038 T27 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/lib
   [junit4]   2> 33038 T27 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/lib).
   [junit4]   2> 33057 T27 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 600000
   [junit4]   2> 33057 T27 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 33058 T27 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 60000
   [junit4]   2> 33058 T27 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 33058 T27 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 33059 T27 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 33059 T27 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 33060 T27 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 33060 T27 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 33061 T27 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 33061 T27 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 33070 T27 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=30000&connTimeout=30000&retry=true
   [junit4]   2> 33072 T27 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 33072 T27 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 33073 T27 oasc.CoreContainer.load Node Name: testNode
   [junit4]   2> 33076 T27 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, schema=schema.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, name=collection1, collection=collection1, instanceDir=collection1, transient=false, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001, loadOnStartup=true, shard=shard1}
   [junit4]   2> 33077 T28 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 33078 T28 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 33079 T28 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 33080 T28 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 33203 T28 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 33241 T28 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 33322 T28 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 33323 T28 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 33387 T28 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 33971 T28 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 34010 T28 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 34015 T28 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 34056 T28 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 34065 T28 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 34073 T28 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 34075 T28 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 34075 T28 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 34076 T28 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 34077 T28 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 34078 T28 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 34078 T28 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 34079 T28 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 34091 T28 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/
   [junit4]   2> 34101 T28 c:collection1 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 34187 T28 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001
   [junit4]   2> 34188 T28 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/index/
   [junit4]   2> 34188 T28 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/index' doesn't exist. Creating new index...
   [junit4]   2> 34189 T28 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/index
   [junit4]   2> 34190 T28 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 34199 T28 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_1,generation=1}
   [junit4]   2> 34200 T28 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 34614 T28 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 34751 T28 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 34761 T28 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 34762 T28 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 34762 T28 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 34763 T28 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 34763 T28 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 34816 T28 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 34816 T28 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 35079 T28 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 35080 T28 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 35081 T28 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 35108 T28 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 35110 T28 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 35112 T28 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 35114 T28 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 35373 T28 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 35480 T28 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /get,/admin/file,/admin/threads,standard,/search-facet-def,spellCheckCompRH1,spellCheckCompRH,/update/json/docs,/search-facet-invariants,/update/json,/update,dismax,/terms,mltrh,/replication,/admin/system,/schema,spellCheckCompRH_Direct,spellCheckWithWordbreak_Direct,/update/csv,/admin/plugins,/admin/ping,/admin/properties,/admin/segments,/admin/luke,defaults,/admin/logging,tvrh,/admin/mbeans,/config,lazy,/debug/dump,/mlt,spellCheckWithWordbreak,mock
   [junit4]   2> 35489 T28 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 35491 T28 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 35584 T28 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 35585 T28 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 35634 T28 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 35635 T28 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 35635 T28 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=14, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6545133281336737]
   [junit4]   2> 35637 T28 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_1,generation=1}
   [junit4]   2> 35637 T28 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 35638 T28 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@41341e7[collection1] main
   [junit4]   2> 35668 T28 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 35669 T28 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 35669 T28 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 35670 T28 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 35913 T28 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 36315 T28 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 36316 T29 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 36319 T29 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 36319 T29 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 36319 T29 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 36321 T29 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 36323 T29 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 36324 T29 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 36324 T28 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 36325 T29 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 36326 T29 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 36326 T29 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@41341e7[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 36346 T27 oas.SolrTestCaseJ4.initCore ####initCore end
   [junit4]   2> 36355 T27 oas.SolrTestCaseJ4.setUp ###Starting testSoftAndHardCommitMaxTimeMixedAdds
   [junit4]   2> 36356 T27 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 36357 T27 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 36358 T27 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 36359 T27 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 36537 T27 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 36575 T27 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 36648 T27 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 36649 T27 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 36706 T27 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 37138 T27 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 37161 T27 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 37163 T27 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 37186 T27 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 37192 T27 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 37196 T27 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 37197 T27 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 37198 T27 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 37198 T27 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 37199 T27 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 37200 T27 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 37200 T27 oasc.CoreContainer.reload Reloading SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 37201 T27 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/
   [junit4]   2> 37201 T27 c:collection1 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 37202 T27 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/index/
   [junit4]   2> 37206 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 37207 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 37208 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 37208 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 37208 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 37209 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 37209 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 37210 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 37210 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 37211 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 37212 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 37212 T27 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 37233 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 37236 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 37238 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 37240 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 37243 T27 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 37245 T27 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /get,/admin/file,/admin/threads,standard,/search-facet-def,spellCheckCompRH1,spellCheckCompRH,/update/json/docs,/search-facet-invariants,/update/json,/update,dismax,/terms,mltrh,/replication,/admin/system,/schema,spellCheckCompRH_Direct,spellCheckWithWordbreak_Direct,/update/csv,/admin/plugins,/admin/ping,/admin/properties,/admin/segments,/admin/luke,defaults,/admin/logging,tvrh,/admin/mbeans,/config,lazy,/debug/dump,/mlt,spellCheckWithWordbreak,mock
   [junit4]   2> 37247 T27 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 37250 T27 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 37256 T27 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 37257 T27 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 37258 T27 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@3dd2c849[collection1] main
   [junit4]   2> 37259 T27 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 37259 T27 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 37259 T27 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 37260 T27 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 37262 T27 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 37274 T27 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 37276 T31 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 37276 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
   [junit4]   2> 37277 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
   [junit4]   2> 37277 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Closing old IndexWriter... core=collection1
   [junit4]   2> 37277 T31 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 37278 T31 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 37278 T27 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=20, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.858064534025872]
   [junit4]   2> 37278 T31 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 37280 T31 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 37281 T31 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 37282 T31 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 37283 T31 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 37284 T31 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 37285 T31 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3dd2c849[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 37287 T27 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_1,generation=1}
   [junit4]   2> 37287 T27 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 37288 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
   [junit4]   2> 37288 T27 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@442cca32[collection1] main
   [junit4]   2> 37300 T27 c:collection1 oasc.CoreContainer.registerCore replacing core: collection1
   [junit4]   2> 37300 T27 c:collection1 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@367c10a8
   [junit4]   2> 37301 T31 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@442cca32[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 37301 T27 c:collection1 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=0,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=0,transaction_logs_total_number=0}
   [junit4]   2> 37302 T27 c:collection1 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> ASYNC  NEW_CORE C3 name=collection1 org.apache.solr.core.SolrCore@469681e0
   [junit4]   2> 38767 T27 c:collection1 C3 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[529 (1497931605250932736)]} 0 1458
   [junit4]   2> 39257 T33 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 39796 T27 c:collection1 oas.SolrTestCaseJ4.tearDown ###Ending testSoftAndHardCommitMaxTimeMixedAdds
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SoftAutoCommitTest -Dtests.method=testSoftAndHardCommitMaxTimeMixedAdds -Dtests.seed=BB89623F8C2FAD9B -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=el_CY -Dtests.timezone=Europe/Luxembourg -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE 3.54s J2 | SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: soft529 wasn't fast enough
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([BB89623F8C2FAD9B:EA5D9BBF3D5C9D3C]:0)
   [junit4]    > 	at org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds(SoftAutoCommitTest.java:111)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 39893 T27 c:collection1 oas.SolrTestCaseJ4.setUp ###Starting testSoftAndHardCommitMaxTimeRapidAdds
   [junit4]   2> 39893 T27 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 39895 T27 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 39896 T27 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 39897 T27 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 39957 T32 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 40080 T27 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 40134 T27 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 40228 T27 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 40229 T27 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 40284 T33 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4e096ef3[collection1] main
   [junit4]   2> 40285 T33 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 40286 T31 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4e096ef3[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):c1)))}
   [junit4]   2> 40287 T32 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_1,generation=1}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_2,generation=2}
   [junit4]   2> 40287 T32 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 40288 T32 c:collection1 oasc.SolrCore.openNewSearcher SolrIndexSearcher has not changed - not re-opening: org.apache.solr.search.SolrIndexSearcher
   [junit4]   2> 40289 T32 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 40321 T27 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 40879 T27 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 40915 T27 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 40920 T27 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 40955 T27 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 40963 T27 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 40970 T27 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 40972 T27 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 40973 T27 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 40973 T27 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 40975 T27 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 40975 T27 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 40976 T27 c:collection1 oasc.CoreContainer.reload Reloading SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 40977 T27 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/
   [junit4]   2> 40978 T27 c:collection1 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 40978 T27 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/index/
   [junit4]   2> 40983 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 40984 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 40985 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 40986 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 40986 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 40987 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 40988 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 40990 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 40990 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 40992 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 40992 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 40993 T27 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 41030 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 41033 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 41038 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 41041 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 41047 T27 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 41049 T27 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /get,/admin/file,/admin/threads,standard,/search-facet-def,spellCheckCompRH1,spellCheckCompRH,/update/json/docs,/search-facet-invariants,/update/json,/update,dismax,/terms,mltrh,/replication,/admin/system,/schema,spellCheckCompRH_Direct,spellCheckWithWordbreak_Direct,/update/csv,/admin/plugins,/admin/ping,/admin/properties,/admin/segments,/admin/luke,defaults,/admin/logging,tvrh,/admin/mbeans,/config,lazy,/debug/dump,/mlt,spellCheckWithWordbreak,mock
   [junit4]   2> 41052 T27 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 41055 T27 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 41056 T27 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 41056 T27 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 41058 T27 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@e611d79[collection1] main
   [junit4]   2> 41059 T27 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 41060 T27 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 41060 T27 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 41061 T27 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 41065 T27 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 41083 T27 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 41085 T34 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 41085 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
   [junit4]   2> 41086 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
   [junit4]   2> 41086 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Closing old IndexWriter... core=collection1
   [junit4]   2> 41087 T34 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 41088 T34 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 41088 T34 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 41090 T34 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 41090 T27 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1831758711, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 41091 T34 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 41093 T34 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 41095 T34 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 41095 T27 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_2,generation=2}
   [junit4]   2> 41096 T27 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 41096 T34 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 41097 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
   [junit4]   2> 41097 T34 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@e611d79[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):c1)))}
   [junit4]   2> 41103 T27 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@6067a790[collection1] main
   [junit4]   2> 41105 T27 c:collection1 oasc.CoreContainer.registerCore replacing core: collection1
   [junit4]   2> 41105 T34 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6067a790[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):c1)))}
   [junit4]   2> 41105 T27 c:collection1 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@469681e0
   [junit4]   2> 41108 T27 c:collection1 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=2,autocommit maxTime=1200ms,autocommits=1,soft autocommit maxTime=500ms,soft autocommits=1,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=1,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=143,transaction_logs_total_number=1}
   [junit4]   2> 41109 T27 c:collection1 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> ASYNC  NEW_CORE C4 name=collection1 org.apache.solr.core.SolrCore@14d97f8d
   [junit4]   2> 41117 T27 c:collection1 C4 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5000 (1497931608497324032)]} 0 6
   [junit4]   2> 41122 T27 c:collection1 C4 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5001 (1497931608506761216)]} 0 2
   [junit4]   2> 41126 T27 c:collection1 C4 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5002 (1497931608512004096)]} 0 2
   [junit4]   2> 41130 T27 c:collection1 C4 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5003 (1497931608516198400)]} 0 1
   [junit4]   2> 41135 T27 c:collection1 C4 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5004 (1497931608520392704)]} 0 1
   [junit4]   2> 41137 T27 c:collection1 oas.SolrTestCaseJ4.tearDown ###Ending testSoftAndHardCommitMaxTimeRapidAdds
   [junit4]   2> 41152 T27 c:collection1 oas.SolrTestCaseJ4.setUp ###Starting testSoftAndHardCommitMaxTimeDelete
   [junit4]   2> 41152 T27 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 41155 T27 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 41156 T27 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 41157 T27 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 41313 T27 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 41363 T27 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 41458 T27 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 41459 T27 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 41498 T27 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 41618 T36 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 41783 T36 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@67d122a8[collection1] main
   [junit4]   2> 41784 T36 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 41784 T34 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@67d122a8[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):c1) Uninverting(_1(5.2.0):c5)))}
   [junit4]   2> 42043 T27 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 42095 T27 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 42100 T27 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 42130 T27 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 42142 T27 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 42148 T27 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 42149 T27 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 42150 T27 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 42151 T27 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 42152 T27 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 42152 T27 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 42153 T27 c:collection1 oasc.CoreContainer.reload Reloading SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 42154 T27 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/
   [junit4]   2> 42155 T27 c:collection1 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 42156 T27 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/index/
   [junit4]   2> 42160 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 42161 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 42162 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 42163 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 42163 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 42164 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 42165 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 42165 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 42166 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 42167 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 42168 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 42169 T27 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 42196 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 42199 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 42202 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 42205 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 42210 T27 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 42213 T27 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /get,/admin/file,/admin/threads,standard,/search-facet-def,spellCheckCompRH1,spellCheckCompRH,/update/json/docs,/search-facet-invariants,/update/json,/update,dismax,/terms,mltrh,/replication,/admin/system,/schema,spellCheckCompRH_Direct,spellCheckWithWordbreak_Direct,/update/csv,/admin/plugins,/admin/ping,/admin/properties,/admin/segments,/admin/luke,defaults,/admin/logging,tvrh,/admin/mbeans,/config,lazy,/debug/dump,/mlt,spellCheckWithWordbreak,mock
   [junit4]   2> 42216 T27 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 42218 T27 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 42224 T27 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 42225 T27 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 42226 T27 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@22bf06d6[collection1] main
   [junit4]   2> 42227 T27 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 42228 T27 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 42228 T27 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 42228 T27 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 42231 T27 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 42242 T27 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 42243 T37 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 42243 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
   [junit4]   2> 42244 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
   [junit4]   2> 42245 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Closing old IndexWriter... core=collection1
   [junit4]   2> 42245 T37 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 42246 T37 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 42246 T27 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_2,generation=2}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_3,generation=3}
   [junit4]   2> 42246 T37 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 42247 T27 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
   [junit4]   2> 42248 T27 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=16, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7145006735137542]
   [junit4]   2> 42248 T37 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 42250 T37 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 42252 T37 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 42252 T27 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_3,generation=3}
   [junit4]   2> 42253 T27 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
   [junit4]   2> 42253 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
   [junit4]   2> 42253 T37 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 42254 T27 c:collection1 oasc.SolrCore.getSearcher WARN [collection1] PERFORMANCE WARNING: Overlapping onDeckSearchers=2
   [junit4]   2> 42255 T37 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 42256 T37 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@22bf06d6[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):c1) Uninverting(_1(5.2.0):c5)))}
   [junit4]   2> 42260 T27 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@710b47d2[collection1] main
   [junit4]   2> 42261 T27 c:collection1 oasc.CoreContainer.registerCore replacing core: collection1
   [junit4]   2> 42261 T37 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@710b47d2[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):c1) Uninverting(_1(5.2.0):c5)))}
   [junit4]   2> 42262 T27 c:collection1 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@14d97f8d
   [junit4]   2> 42263 T27 c:collection1 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=1,autocommit maxTime=1200ms,autocommits=0,soft autocommit maxTime=500ms,soft autocommits=1,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=5,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=5,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=143,transaction_logs_total_number=1}
   [junit4]   2> 42264 T27 c:collection1 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> ASYNC  NEW_CORE C5 name=collection1 org.apache.solr.core.SolrCore@16552add
   [junit4]   2> 42268 T27 c:collection1 C5 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[529 (1497931609708429312)]} 0 3
   [junit4]   2> 42271 T27 c:collection1 C5 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 42304 T27 c:collection1 C5 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_3,generation=3}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_4,generation=4}
   [junit4]   2> 42304 T27 c:collection1 C5 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 4
   [junit4]   2> 42306 T27 c:collection1 C5 oass.SolrIndexSearcher.<init> Opening Searcher@28fbf6a1[collection1] main
   [junit4]   2> 42306 T27 c:collection1 C5 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 42307 T37 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@28fbf6a1[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(5.2.0):c5) Uninverting(_2(5.2.0):c1)))}
   [junit4]   2> 42309 T27 c:collection1 C5 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 38
   [junit4]   2> 42315 T27 c:collection1 C5 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {delete=[529 (-1497931609756663808)]} 0 4
   [junit4]   2> 42816 T39 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 42826 T39 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@7a3335c5[collection1] main
   [junit4]   2> 42827 T39 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 42827 T37 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7a3335c5[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(5.2.0):c5)))}
   [junit4]   2> 42830 T27 c:collection1 C5 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[550 (1497931610297729024)]} 0 3
   [junit4]   2> 43331 T39 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 43349 T39 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@536fe809[collection1] main
   [junit4]   2> 43350 T39 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 43350 T37 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@536fe809[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(5.2.0):c5) Uninverting(_3(5.2.0):c1)))}
   [junit4]   2> 43516 T38 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 43518 T38 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_4,generation=4}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_5,generation=5}
   [junit4]   2> 43518 T38 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 5
   [junit4]   2> 43520 T38 c:collection1 oasc.SolrCore.openNewSearcher SolrIndexSearcher has not changed - not re-opening: org.apache.solr.search.SolrIndexSearcher
   [junit4]   2> 43520 T38 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 45521 T27 c:collection1 oas.SolrTestCaseJ4.tearDown ###Ending testSoftAndHardCommitMaxTimeDelete
   [junit4]   2> 45523 T27 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> 45524 T27 c:collection1 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=1465666313
   [junit4]   2> 45525 T27 c:collection1 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@16552add
   [junit4]   2> 45526 T27 c:collection1 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=4,autocommit maxTime=1200ms,autocommits=1,soft autocommit maxTime=500ms,soft autocommits=2,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=2,cumulative_deletesById=1,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=707,transaction_logs_total_number=3}
   [junit4]   2> 45527 T27 c:collection1 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 45528 T27 c:collection1 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 45529 T27 c:collection1 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 45530 T27 c:collection1 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 45533 T27 c:collection1 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 45533 T27 c:collection1 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001 [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001;done=false>>]
   [junit4]   2> 45534 T27 c:collection1 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001
   [junit4]   2> 45535 T27 c:collection1 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/index;done=false>>]
   [junit4]   2> 45535 T27 c:collection1 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/index
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50), sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=el_CY, timezone=Europe/Luxembourg
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=210408840,total=255852544
   [junit4]   2> NOTE: All tests run in this JVM: [TestManagedSchemaFieldTypeResource, SoftAutoCommitTest]
   [junit4] Completed on J2 in 15.17s, 3 tests, 1 failure <<< FAILURES!

[...truncated 1512 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 480 suites, 1910 tests, 1 failure, 62 ignored (22 assumptions)

Total time: 60 minutes 25 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 221032214 bytes
Compression is 0.0%
Took 39 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2913 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2913/

2 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest

Error Message:
ERROR: SolrIndexSearcher opens=8 closes=7

Stack Trace:
java.lang.AssertionError: ERROR: SolrIndexSearcher opens=8 closes=7
	at __randomizedtesting.SeedInfo.seed([4868EF3346E25AEB]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.SolrTestCaseJ4.endTrackingSearchers(SolrTestCaseJ4.java:496)
	at org.apache.solr.SolrTestCaseJ4.afterClass(SolrTestCaseJ4.java:232)
	at sun.reflect.GeneratedMethodAccessor35.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:799)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest

Error Message:
2 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest:     1) Thread[id=3276, name=searcherExecutor-2038-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=3262, name=qtp1606563175-3262, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:503)         at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient.blockUntilFinished(ConcurrentUpdateSolrClient.java:389)         at org.apache.solr.update.StreamingSolrClients.blockUntilFinished(StreamingSolrClients.java:103)         at org.apache.solr.update.SolrCmdDistributor.blockAndDoRetries(SolrCmdDistributor.java:229)         at org.apache.solr.update.SolrCmdDistributor.finish(SolrCmdDistributor.java:89)         at org.apache.solr.update.processor.DistributedUpdateProcessor.doFinish(DistributedUpdateProcessor.java:781)         at org.apache.solr.update.processor.DistributedUpdateProcessor.finish(DistributedUpdateProcessor.java:1643)         at org.apache.solr.update.processor.LogUpdateProcessor.finish(LogUpdateProcessorFactory.java:183)         at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:83)         at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:143)         at org.apache.solr.core.SolrCore.execute(SolrCore.java:1988)         at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:828)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:446)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)         at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:102)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)         at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)         at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)         at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)         at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)         at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)         at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)         at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)         at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)         at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)         at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)         at org.eclipse.jetty.server.Server.handle(Server.java:368)         at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)         at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)         at org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:953)         at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1014)         at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:953)         at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240)         at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)         at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest: 
   1) Thread[id=3276, name=searcherExecutor-2038-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=3262, name=qtp1606563175-3262, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:503)
        at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient.blockUntilFinished(ConcurrentUpdateSolrClient.java:389)
        at org.apache.solr.update.StreamingSolrClients.blockUntilFinished(StreamingSolrClients.java:103)
        at org.apache.solr.update.SolrCmdDistributor.blockAndDoRetries(SolrCmdDistributor.java:229)
        at org.apache.solr.update.SolrCmdDistributor.finish(SolrCmdDistributor.java:89)
        at org.apache.solr.update.processor.DistributedUpdateProcessor.doFinish(DistributedUpdateProcessor.java:781)
        at org.apache.solr.update.processor.DistributedUpdateProcessor.finish(DistributedUpdateProcessor.java:1643)
        at org.apache.solr.update.processor.LogUpdateProcessor.finish(LogUpdateProcessorFactory.java:183)
        at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:83)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:143)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:1988)
        at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:828)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:446)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:102)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
        at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
        at org.eclipse.jetty.server.Server.handle(Server.java:368)
        at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
        at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
        at org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:953)
        at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1014)
        at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:953)
        at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240)
        at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
        at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([4868EF3346E25AEB]:0)




Build Log:
[...truncated 10048 lines...]
   [junit4] Suite: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/init-core-data-001
   [junit4]   2> 1172756 T3222 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 1172765 T3222 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1172765 T3223 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1172865 T3222 oasc.ZkTestServer.run start zk server on port:33268
   [junit4]   2> 1172866 T3222 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1172867 T3222 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1172873 T3230 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1a882b79 name:ZooKeeperConnection Watcher:127.0.0.1:33268 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1172873 T3222 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1172874 T3222 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1172874 T3222 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1172878 T3222 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1172879 T3222 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1172881 T3233 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5e5fc8b5 name:ZooKeeperConnection Watcher:127.0.0.1:33268/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1172881 T3222 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1172882 T3222 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1172882 T3222 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1172884 T3222 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1172886 T3222 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1172887 T3222 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1172888 T3222 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1172889 T3222 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1172891 T3222 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 1172892 T3222 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1172995 T3222 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1172996 T3222 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1172998 T3222 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1172998 T3222 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1173000 T3222 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1173000 T3222 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1173002 T3222 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1173002 T3222 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1173004 T3222 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1173005 T3222 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1173006 T3222 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1173007 T3222 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1173008 T3222 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1173009 T3222 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1173010 T3222 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1173011 T3222 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1173012 T3222 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1173013 T3222 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1174614 T3222 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1
   [junit4]   2> 1174621 T3222 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1174628 T3222 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:33280
   [junit4]   2> 1174628 T3222 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=33280, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/tempDir-001/control/data}
   [junit4]   2> 1174629 T3222 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@749cd006
   [junit4]   2> 1174630 T3222 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/'
   [junit4]   2> 1174700 T3222 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/solr.xml
   [junit4]   2> 1174835 T3222 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores
   [junit4]   2> 1174836 T3222 oasc.CoreContainer.<init> New CoreContainer 930892105
   [junit4]   2> 1174837 T3222 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/]
   [junit4]   2> 1174837 T3222 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/lib
   [junit4]   2> 1174839 T3222 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/lib).
   [junit4]   2> 1174856 T3222 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1174857 T3222 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1174857 T3222 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1174858 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1174858 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1174859 T3222 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1174859 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1174860 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1174860 T3222 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1174861 T3222 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1174861 T3222 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1174879 T3222 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1174881 T3222 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1174882 T3222 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1174882 T3222 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1174883 T3222 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:33268/solr
   [junit4]   2> 1174883 T3222 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1174884 T3222 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1174885 T3222 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1174888 T3245 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1fd45422 name:ZooKeeperConnection Watcher:127.0.0.1:33268 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1174888 T3222 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1174889 T3222 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1174894 T3222 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1174897 T3248 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@65435e59 name:ZooKeeperConnection Watcher:127.0.0.1:33268/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1174897 T3222 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1174901 T3222 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1174904 T3222 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1174907 T3222 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1174910 T3222 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1174915 T3222 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1174917 T3222 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1174919 T3222 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1174921 T3222 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1174923 T3222 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33280_
   [junit4]   2> 1174925 T3222 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:33280_
   [junit4]   2> 1174927 T3222 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1174930 T3222 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1174932 T3222 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1174934 T3222 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:33280_
   [junit4]   2> 1174935 T3222 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1174937 T3222 oasc.Overseer.start Overseer (id=93619858301714435-127.0.0.1:33280_-n_0000000000) starting
   [junit4]   2> 1174939 T3222 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1174948 T3222 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1174950 T3250 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1174950 T3222 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1174950 T3249 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1174957 T3222 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores
   [junit4]   2> 1174967 T3222 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {schema=schema.xml, config=solrconfig.xml, transient=false, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1, name=collection1, shard=, loadOnStartup=true, collection=control_collection, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1/, dataDir=data/}
   [junit4]   2> 1174968 T3222 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1/
   [junit4]   2> 1174969 T3222 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1174970 T3252 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1174970 T3252 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1174972 T3248 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1174972 T3252 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1174973 T3249 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33280",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:33280_",
   [junit4]   2> 	  "roles":null} current state version: 0
   [junit4]   2> 1174974 T3249 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33280",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:33280_",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 1174974 T3249 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1174974 T3249 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1174976 T3248 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1175972 T3252 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1175973 T3252 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1175974 T3252 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1175974 T3252 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1175976 T3252 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1175977 T3252 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1/'
   [junit4]   2> 1176004 T3252 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1176008 T3252 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1176031 T3252 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1176086 T3252 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1176176 T3252 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1176279 T3252 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1176291 T3252 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1176758 T3252 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1176761 T3252 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1176763 T3252 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1176768 T3252 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1176796 T3252 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1176797 T3252 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
   [junit4]   2> 1176801 T3252 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1/data/
   [junit4]   2> 1176802 T3252 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2a7655aa
   [junit4]   2> 1176806 T3252 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1/data
   [junit4]   2> 1176810 T3252 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1/data/index/
   [junit4]   2> 1176813 T3252 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1176815 T3252 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1/data/index
   [junit4]   2> 1176816 T3252 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=31, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=32.4326171875, floorSegmentMB=0.822265625, forceMergeDeletesPctAllowed=24.40693562206468, segmentsPerTier=33.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6199579108735648
   [junit4]   2> 1176826 T3252 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 1176826 T3252 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1176836 T3252 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1176837 T3252 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1176837 T3252 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1176837 T3252 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1176838 T3252 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1176838 T3252 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1176839 T3252 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1176839 T3252 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1176840 T3252 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1176903 T3252 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1176908 T3252 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1176914 T3252 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1176917 T3252 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1176924 T3252 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /config,/admin/threads,standard,/admin/logging,/admin/luke,/update/json,/update/json/docs,/admin/ping,/admin/file,/admin/mbeans,/admin/system,/update/csv,/admin/plugins,/replication,/get,/admin/properties,/update,/admin/segments,/schema
   [junit4]   2> 1176929 T3252 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1176930 T3252 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1176930 T3252 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1176934 T3252 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1176935 T3252 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1176937 T3252 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=36, maxMergeAtOnceExplicit=10, maxMergedSegmentMB=72.5986328125, floorSegmentMB=1.576171875, forceMergeDeletesPctAllowed=29.48913303940835, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.892008954113685
   [junit4]   2> 1176939 T3252 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 1176940 T3252 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1176941 T3252 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@6cec9e83[collection1] main
   [junit4]   2> 1176941 T3252 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1176943 T3252 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1176943 T3252 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1176944 T3252 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1176944 T3252 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1176945 T3252 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1176945 T3252 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1176946 T3252 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1176946 T3252 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1176947 T3252 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1176948 T3253 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6cec9e83[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1176949 T3252 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1176952 T3256 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:33280 collection:control_collection shard:shard1
   [junit4]   2> 1176954 T3222 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 1176954 T3222 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1176955 T3256 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1176958 T3222 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1176959 T3222 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1176962 T3259 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@40d62081 name:ZooKeeperConnection Watcher:127.0.0.1:33268/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1176962 T3222 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1176963 T3222 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1176964 T3222 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1176970 T3222 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1176971 T3256 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1176974 T3248 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1176974 T3256 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1176974 T3256 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C532 name=collection1 org.apache.solr.core.SolrCore@66c2b8de url=http://127.0.0.1:33280/collection1 node=127.0.0.1:33280_ C532_STATE=coll:control_collection core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:33280, node_name=127.0.0.1:33280_}
   [junit4]   2> 1176975 T3256 C:control_collection S:shard1 C532 P33280 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:33280/collection1/
   [junit4]   2> 1176975 T3249 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 1176975 T3256 C:control_collection S:shard1 C532 P33280 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1176976 T3256 C:control_collection S:shard1 C532 P33280 oasc.SyncStrategy.syncToMe http://127.0.0.1:33280/collection1/ has no replicas
   [junit4]   2> 1176976 T3256 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:33280/collection1/ shard1
   [junit4]   2> 1176976 T3256 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1176978 T3259 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1176978 T3248 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1176984 T3248 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1176986 T3249 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33280",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 1177091 T3248 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1177092 T3259 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1177139 T3256 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:33280/collection1/ and leader is http://127.0.0.1:33280/collection1/
   [junit4]   2> 1177140 T3256 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:33280
   [junit4]   2> 1177140 T3256 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1177140 T3256 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1177140 T3256 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1177143 T3248 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1177145 T3249 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33280",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:33280_",
   [junit4]   2> 	  "roles":null} current state version: 3
   [junit4]   2> 1177146 T3249 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33280",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:33280_",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 1177251 T3248 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1177251 T3259 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1180447 T3222 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1
   [junit4]   2> 1180453 T3222 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001
   [junit4]   2> 1180454 T3222 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1180461 T3222 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:33284
   [junit4]   2> 1180461 T3222 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores, solrconfig=solrconfig.xml, hostContext=/, hostPort=33284}
   [junit4]   2> 1180462 T3222 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@749cd006
   [junit4]   2> 1180462 T3222 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/'
   [junit4]   2> 1180551 T3222 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/solr.xml
   [junit4]   2> 1180683 T3222 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores
   [junit4]   2> 1180683 T3222 oasc.CoreContainer.<init> New CoreContainer 863431378
   [junit4]   2> 1180684 T3222 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/]
   [junit4]   2> 1180684 T3222 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/lib
   [junit4]   2> 1180687 T3222 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/lib).
   [junit4]   2> 1180702 T3222 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1180702 T3222 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1180703 T3222 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1180703 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1180703 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1180704 T3222 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1180704 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1180705 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1180705 T3222 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1180705 T3222 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1180706 T3222 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1180725 T3222 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1180727 T3222 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1180727 T3222 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1180727 T3222 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1180728 T3222 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:33268/solr
   [junit4]   2> 1180728 T3222 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1180729 T3222 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1180730 T3222 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1180733 T3271 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@254dca1c name:ZooKeeperConnection Watcher:127.0.0.1:33268 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1180733 T3222 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1180734 T3222 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1180739 T3222 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1180741 T3274 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6a2ea17c name:ZooKeeperConnection Watcher:127.0.0.1:33268/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1180742 T3222 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1180751 T3222 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1181759 T3222 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33284_
   [junit4]   2> 1181760 T3222 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:33284_
   [junit4]   2> 1181764 T3222 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1181766 T3222 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores
   [junit4]   2> 1181768 T3222 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1/, schema=schema.xml, coreNodeName=, transient=false, shard=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1, config=solrconfig.xml, dataDir=data/, name=collection1, loadOnStartup=true}
   [junit4]   2> 1181769 T3222 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1/
   [junit4]   2> 1181769 T3222 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1181770 T3275 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1181770 T3275 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1181771 T3275 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1181771 T3248 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1181772 T3249 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33284",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:33284_",
   [junit4]   2> 	  "roles":null} current state version: 4
   [junit4]   2> 1181772 T3249 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33284",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:33284_",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 1181773 T3249 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 1181773 T3249 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1181774 T3274 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1181774 T3259 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1181774 T3248 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1182771 T3275 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1182772 T3275 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1182773 T3275 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1182773 T3275 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1182775 T3275 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1182776 T3275 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1/'
   [junit4]   2> 1182802 T3275 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1182807 T3275 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1182825 T3275 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1182868 T3275 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1182952 T3275 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1183054 T3275 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1183074 T3275 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1183668 T3275 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1183674 T3275 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1183677 T3275 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1183686 T3275 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1183718 T3275 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1183718 T3275 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
   [junit4]   2> 1183721 T3275 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 1183721 T3275 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2a7655aa
   [junit4]   2> 1183724 T3275 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1/data
   [junit4]   2> 1183725 T3275 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 1183726 T3275 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1183728 T3275 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 1183728 T3275 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=31, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=32.4326171875, floorSegmentMB=0.822265625, forceMergeDeletesPctAllowed=24.40693562206468, segmentsPerTier=33.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6199579108735648
   [junit4]   2> 1183735 T3275 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 1183736 T3275 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1183748 T3275 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1183748 T3275 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1183749 T3275 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1183749 T3275 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1183750 T3275 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1183750 T3275 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1183751 T3275 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1183751 T3275 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1183752 T3275 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1183811 T3275 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1183817 T3275 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1183822 T3275 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1183828 T3275 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1183844 T3275 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /config,/admin/threads,standard,/admin/logging,/admin/luke,/update/json,/update/json/docs,/admin/ping,/admin/file,/admin/mbeans,/admin/system,/update/csv,/admin/plugins,/replication,/get,/admin/properties,/update,/admin/segments,/schema
   [junit4]   2> 1183849 T3275 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1183850 T3275 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1183850 T3275 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1183854 T3275 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1183855 T3275 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1183858 T3275 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=36, maxMergeAtOnceExplicit=10, maxMergedSegmentMB=72.5986328125, floorSegmentMB=1.576171875, forceMergeDeletesPctAllowed=29.48913303940835, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.892008954113685
   [junit4]   2> 1183860 T3275 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 1183861 T3275 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1183862 T3275 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@7b2b64a1[collection1] main
   [junit4]   2> 1183862 T3275 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1183863 T3275 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1183864 T3275 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1183864 T3275 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1183865 T3275 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1183865 T3275 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1183866 T3275 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1183866 T3275 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1183867 T3275 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1183867 T3275 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1183869 T3276 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7b2b64a1[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1183869 T3275 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1183871 T3279 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:33284 collection:collection1 shard:shard1
   [junit4]   2> 1183872 T3222 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 1183872 T3279 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1183872 T3222 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1183879 T3279 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1183883 T3248 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1183884 T3279 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1183884 T3279 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1183884 T3249 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> ASYNC  NEW_CORE C533 name=collection1 org.apache.solr.core.SolrCore@6d5604b6 url=http://127.0.0.1:33284/collection1 node=127.0.0.1:33284_ C533_STATE=coll:collection1 core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:33284, node_name=127.0.0.1:33284_}
   [junit4]   2> 1183884 T3279 C:collection1 S:shard1 C533 P33284 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:33284/collection1/
   [junit4]   2> 1183885 T3279 C:collection1 S:shard1 C533 P33284 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1183885 T3279 C:collection1 S:shard1 C533 P33284 oasc.SyncStrategy.syncToMe http://127.0.0.1:33284/collection1/ has no replicas
   [junit4]   2> 1183885 T3279 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:33284/collection1/ shard1
   [junit4]   2> 1183886 T3279 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1183886 T3274 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1183886 T3259 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1183886 T3248 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1183891 T3248 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1183894 T3249 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33284",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 1183999 T3274 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1183999 T3259 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1183999 T3248 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1184044 T3279 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:33284/collection1/ and leader is http://127.0.0.1:33284/collection1/
   [junit4]   2> 1184045 T3279 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:33284
   [junit4]   2> 1184045 T3279 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1184045 T3279 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1184046 T3279 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1184047 T3248 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1184048 T3249 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33284",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:33284_",
   [junit4]   2> 	  "roles":null} current state version: 7
   [junit4]   2> 1184049 T3249 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33284",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:33284_",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 1184154 T3248 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1184154 T3274 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1184154 T3259 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1187313 T3222 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001/cores/collection1
   [junit4]   2> 1187318 T3222 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001
   [junit4]   2> 1187319 T3222 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1187324 T3222 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:33289
   [junit4]   2> 1187325 T3222 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/tempDir-001/jetty2, hostPort=33289, solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001/cores}
   [junit4]   2> 1187326 T3222 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@749cd006
   [junit4]   2> 1187327 T3222 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001/'
   [junit4]   2> 1187444 T3222 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001/solr.xml
   [junit4]   2> 1187581 T3222 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001/cores
   [junit4]   2> 1187582 T3222 oasc.CoreContainer.<init> New CoreContainer 83748162
   [junit4]   2> 1187582 T3222 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001/]
   [junit4]   2> 1187583 T3222 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001/lib
   [junit4]   2> 1187586 T3222 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001/lib).
   [junit4]   2> 1187605 T3222 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1187605 T3222 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1187606 T3222 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1187606 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1187606 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1187607 T3222 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1187607 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1187607 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1187608 T3222 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1187608 T3222 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1187608 T3222 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1187626 T3222 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1187629 T3222 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1187629 T3222 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1187630 T3222 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1187630 T3222 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:33268/solr
   [junit4]   2> 1187631 T3222 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1187631 T3222 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1187632 T3222 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1187636 T3291 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6a348bc2 name:ZooKeeperConnection Watcher:127.0.0.1:33268 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1187636 T3222 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1187637 T3222 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1187642 T3222 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1187644 T3294 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@16ee53d9 name:ZooKeeperConnection Watcher:127.0.0.1:33268/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1187644 T3222 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1187653 T3222 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1188661 T3222 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33289_
   [junit4]   2> 1188663 T3222 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:33289_
   [junit4]   2> 1188669 T3222 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1188674 T3222 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001/cores
   [junit4]   2> 1188685 T3222 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001/cores/collection1, collection=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001/cores/collection1/, schema=schema.xml, transient=false, config=solrconfig.xml, loadOnStartup=true, coreNodeName=, shard=, dataDir=data/}
   [junit4]   2> 1188686 T3222 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/sol

[...truncated too long message...]

rmat(compressionMode=DUMMY, chunkSize=8, maxDocsPerChunk=79, blockSize=578), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=DUMMY, chunkSize=8, blockSize=578)), sim=DefaultSimilarity, locale=sv, timezone=Mexico/BajaSur
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=123053320,total=283115520
   [junit4]   2> NOTE: All tests run in this JVM: [DistribDocExpirationUpdateProcessorTest, StatsComponentTest, TestReload, ExitableDirectoryReaderTest, TestSweetSpotSimilarityFactory, ParsingFieldUpdateProcessorsTest, TimeZoneUtilsTest, TestFieldCollectionResource, CollectionReloadTest, TestSerializedLuceneMatchVersion, TestStressUserVersions, SimpleFacetsTest, HighlighterMaxOffsetTest, TestLRUStatsCache, TestNRTOpen, TestJsonRequest, TestSolr4Spatial2, ZkNodePropsTest, SortByFunctionTest, TestJsonFacets, MoreLikeThisHandlerTest, HdfsUnloadDistributedZkTest, SuggesterFSTTest, TestJmxIntegration, DebugComponentTest, CSVRequestHandlerTest, TestStandardQParsers, TestJettySolrRunner, TestAddFieldRealTimeGet, HdfsChaosMonkeySafeLeaderTest, TestRandomMergePolicy, TermVectorComponentDistributedTest, DistributedQueryComponentCustomSortTest, TestManagedSynonymFilterFactory, TestZkChroot, SOLR749Test, BlockDirectoryTest, TestArbitraryIndexDir, SSLMigrationTest, CursorMarkTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, DeleteReplicaTest, TestSearchPerf, CollectionsAPIDistributedZkTest, TestSolrConfigHandlerConcurrent, DistribCursorPagingTest, RequiredFieldsTest, UUIDFieldTest, XmlUpdateRequestHandlerTest, RegexBoostProcessorTest, URLClassifyProcessorTest, AlternateDirectoryTest, TestCloudInspectUtil, TestSchemaManager, JSONWriterTest, TestManagedSchema, DistribJoinFromCollectionTest, BasicDistributedZk2Test, TestDistributedGrouping, ChaosMonkeyNothingIsSafeTest]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ChaosMonkeyNothingIsSafeTest -Dtests.seed=4868EF3346E25AEB -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=sv -Dtests.timezone=Mexico/BajaSur -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J3 | ChaosMonkeyNothingIsSafeTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: ERROR: SolrIndexSearcher opens=8 closes=7
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([4868EF3346E25AEB]:0)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.endTrackingSearchers(SolrTestCaseJ4.java:496)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.afterClass(SolrTestCaseJ4.java:232)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest: 
   [junit4]    >    1) Thread[id=3276, name=searcherExecutor-2038-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    2) Thread[id=3262, name=qtp1606563175-3262, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at java.lang.Object.wait(Native Method)
   [junit4]    >         at java.lang.Object.wait(Object.java:503)
   [junit4]    >         at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient.blockUntilFinished(ConcurrentUpdateSolrClient.java:389)
   [junit4]    >         at org.apache.solr.update.StreamingSolrClients.blockUntilFinished(StreamingSolrClients.java:103)
   [junit4]    >         at org.apache.solr.update.SolrCmdDistributor.blockAndDoRetries(SolrCmdDistributor.java:229)
   [junit4]    >         at org.apache.solr.update.SolrCmdDistributor.finish(SolrCmdDistributor.java:89)
   [junit4]    >         at org.apache.solr.update.processor.DistributedUpdateProcessor.doFinish(DistributedUpdateProcessor.java:781)
   [junit4]    >         at org.apache.solr.update.processor.DistributedUpdateProcessor.finish(DistributedUpdateProcessor.java:1643)
   [junit4]    >         at org.apache.solr.update.processor.LogUpdateProcessor.finish(LogUpdateProcessorFactory.java:183)
   [junit4]    >         at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:83)
   [junit4]    >         at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:143)
   [junit4]    >         at org.apache.solr.core.SolrCore.execute(SolrCore.java:1988)
   [junit4]    >         at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:828)
   [junit4]    >         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:446)
   [junit4]    >         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)
   [junit4]    >         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
   [junit4]    >         at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:102)
   [junit4]    >         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
   [junit4]    >         at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
   [junit4]    >         at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
   [junit4]    >         at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
   [junit4]    >         at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
   [junit4]    >         at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
   [junit4]    >         at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
   [junit4]    >         at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
   [junit4]    >         at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
   [junit4]    >         at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
   [junit4]    >         at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
   [junit4]    >         at org.eclipse.jetty.server.Server.handle(Server.java:368)
   [junit4]    >         at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
   [junit4]    >         at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
   [junit4]    >         at org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:953)
   [junit4]    >         at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1014)
   [junit4]    >         at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:953)
   [junit4]    >         at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240)
   [junit4]    >         at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
   [junit4]    >         at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([4868EF3346E25AEB]:0)
   [junit4] Completed on J3 in 487.36s, 1 test, 1 failure, 1 error <<< FAILURES!

[...truncated 701 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 480 suites, 1910 tests, 2 suite-level errors, 62 ignored (22 assumptions)

Total time: 59 minutes 45 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 210258778 bytes
Compression is 0.0%
Took 38 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2912 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2912/

All tests passed

Build Log:
[...truncated 28423 lines...]
check-licenses:
     [echo] License check under: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr
 [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/jcl-over-slf4j-1.7.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/jcl-over-slf4j-1.7.6.jar.sha1

 [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/jul-to-slf4j-1.7.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/jul-to-slf4j-1.7.6.jar.sha1
 [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/slf4j-api-1.7.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/slf4j-api-1.7.6.jar.sha1

[...truncated 3 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:90: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:256: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/tools/custom-tasks.xml:62: License check failed. Check the logs.
If you recently modified ivy-versions.properties or any module's ivy.xml,
make sure you run "ant clean-jars jar-checksums" before running precommit.

Total time: 87 minutes 37 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 1 artifacts
Archive block size is 32768
Received 0 blocks and 464 bytes
Compression is 0.0%
Took 12 ms
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2911 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2911/

1 tests failed.
FAILED:  org.apache.solr.cloud.TestRebalanceLeaders.test

Error Message:
No live SolrServers available to handle this request:[http://127.0.0.1:20802/zg, http://127.0.0.1:20781/zg, http://127.0.0.1:20767/zg, http://127.0.0.1:20792/zg, http://127.0.0.1:20743/zg]

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:20802/zg, http://127.0.0.1:20781/zg, http://127.0.0.1:20767/zg, http://127.0.0.1:20792/zg, http://127.0.0.1:20743/zg]
	at __randomizedtesting.SeedInfo.seed([ED4E1CF455B33B43:651A232EFB4F56BB]:0)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:349)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1067)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.TestRebalanceLeaders.issueCommands(TestRebalanceLeaders.java:281)
	at org.apache.solr.cloud.TestRebalanceLeaders.rebalanceLeaderTest(TestRebalanceLeaders.java:108)
	at org.apache.solr.cloud.TestRebalanceLeaders.test(TestRebalanceLeaders.java:74)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:20767/zg: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 {trace=java.lang.AssertionError
	at org.apache.solr.common.util.ByteUtils.UTF8toUTF16(ByteUtils.java:36)
	at org.apache.solr.common.util.ByteUtils.UTF8toUTF16(ByteUtils.java:64)
	at org.apache.solr.common.cloud.ZkStateReader.fromJSON(ZkStateReader.java:137)
	at org.apache.solr.common.cloud.ZkNodeProps.load(ZkNodeProps.java:92)
	at org.apache.solr.cloud.DistributedQueue.containsTaskWithRequestId(DistributedQueue.java:127)
	at org.apache.solr.handler.admin.CollectionsHandler.overseerCollectionQueueContains(CollectionsHandler.java:647)
	at org.apache.solr.handler.admin.CollectionsHandler.handleResponse(CollectionsHandler.java:669)
	at org.apache.solr.handler.admin.CollectionsHandler.handleResponse(CollectionsHandler.java:652)
	at org.apache.solr.handler.admin.CollectionsHandler.rejoinElection(CollectionsHandler.java:447)
	at org.apache.solr.handler.admin.CollectionsHandler.insurePreferredIsLeader(CollectionsHandler.java:362)
	at org.apache.solr.handler.admin.CollectionsHandler.handleBalanceLeaders(CollectionsHandler.java:269)
	at org.apache.solr.handler.admin.CollectionsHandler.handleRequestBody(CollectionsHandler.java:235)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:143)
	at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:782)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:282)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:102)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
	at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
	at org.eclipse.jetty.server.Server.handle(Server.java:368)
	at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
	at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
	at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
	at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
	at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
	at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
	at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
	at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
	at java.lang.Thread.run(Thread.java:745)
,code=500}</title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /zg/admin/collections. Reason:
<pre>    {trace=java.lang.AssertionError
	at org.apache.solr.common.util.ByteUtils.UTF8toUTF16(ByteUtils.java:36)
	at org.apache.solr.common.util.ByteUtils.UTF8toUTF16(ByteUtils.java:64)
	at org.apache.solr.common.cloud.ZkStateReader.fromJSON(ZkStateReader.java:137)
	at org.apache.solr.common.cloud.ZkNodeProps.load(ZkNodeProps.java:92)
	at org.apache.solr.cloud.DistributedQueue.containsTaskWithRequestId(DistributedQueue.java:127)
	at org.apache.solr.handler.admin.CollectionsHandler.overseerCollectionQueueContains(CollectionsHandler.java:647)
	at org.apache.solr.handler.admin.CollectionsHandler.handleResponse(CollectionsHandler.java:669)
	at org.apache.solr.handler.admin.CollectionsHandler.handleResponse(CollectionsHandler.java:652)
	at org.apache.solr.handler.admin.CollectionsHandler.rejoinElection(CollectionsHandler.java:447)
	at org.apache.solr.handler.admin.CollectionsHandler.insurePreferredIsLeader(CollectionsHandler.java:362)
	at org.apache.solr.handler.admin.CollectionsHandler.handleBalanceLeaders(CollectionsHandler.java:269)
	at org.apache.solr.handler.admin.CollectionsHandler.handleRequestBody(CollectionsHandler.java:235)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:143)
	at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:782)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:282)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:102)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
	at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
	at org.eclipse.jetty.server.Server.handle(Server.java:368)
	at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
	at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
	at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
	at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
	at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
	at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
	at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
	at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
	at java.lang.Thread.run(Thread.java:745)
,code=500}</pre></p>
<hr /><i><small>Powered by Jetty://</small></i>
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
</body>
</html>

	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:525)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:338)
	... 48 more




Build Log:
[...truncated 10082 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestRebalanceLeaders
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/init-core-data-001
   [junit4]   2> 1491198 T5962 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 1491198 T5962 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /zg/
   [junit4]   2> 1491203 T5962 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1491204 T5963 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1491304 T5962 oasc.ZkTestServer.run start zk server on port:20738
   [junit4]   2> 1491305 T5962 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1491306 T5962 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1491309 T5970 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@40885ca name:ZooKeeperConnection Watcher:127.0.0.1:20738 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1491309 T5962 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1491310 T5962 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1491310 T5962 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1491312 T5962 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1491313 T5962 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1491315 T5973 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7c823094 name:ZooKeeperConnection Watcher:127.0.0.1:20738/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1491315 T5962 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1491315 T5962 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1491316 T5962 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1491318 T5962 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1491319 T5962 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1491320 T5962 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1491322 T5962 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1491322 T5962 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1491325 T5962 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 1491325 T5962 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1491427 T5962 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1491427 T5962 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1491429 T5962 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1491430 T5962 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1491431 T5962 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1491432 T5962 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1491433 T5962 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1491434 T5962 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1491436 T5962 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1491436 T5962 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1491438 T5962 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1491438 T5962 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1491440 T5962 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1491441 T5962 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1491442 T5962 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1491443 T5962 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1491444 T5962 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1491445 T5962 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1493135 T5962 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores/collection1
   [junit4]   2> 1493140 T5962 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1493145 T5962 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:20743
   [junit4]   2> 1493145 T5962 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/zg, hostPort=20743, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores}
   [junit4]   2> 1493146 T5962 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 1493146 T5962 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/'
   [junit4]   2> 1493205 T5962 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/solr.xml
   [junit4]   2> 1493272 T5962 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores
   [junit4]   2> 1493273 T5962 oasc.CoreContainer.<init> New CoreContainer 671212554
   [junit4]   2> 1493273 T5962 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/]
   [junit4]   2> 1493273 T5962 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/lib
   [junit4]   2> 1493274 T5962 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/lib).
   [junit4]   2> 1493283 T5962 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1493284 T5962 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1493284 T5962 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1493284 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1493284 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1493285 T5962 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1493285 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1493285 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1493285 T5962 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1493285 T5962 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1493286 T5962 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1493293 T5962 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1493294 T5962 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1493294 T5962 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1493295 T5962 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1493295 T5962 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:20738/solr
   [junit4]   2> 1493295 T5962 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1493295 T5962 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1493296 T5962 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1493297 T5985 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3e33214f name:ZooKeeperConnection Watcher:127.0.0.1:20738 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1493298 T5962 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1493299 T5962 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1493301 T5962 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1493302 T5988 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6718d986 name:ZooKeeperConnection Watcher:127.0.0.1:20738/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1493302 T5962 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1493303 T5962 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1493304 T5962 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1493305 T5962 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1493307 T5962 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1493308 T5962 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1493309 T5962 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1493310 T5962 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1493311 T5962 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1493312 T5962 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:20743_zg
   [junit4]   2> 1493312 T5962 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:20743_zg
   [junit4]   2> 1493313 T5962 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1493314 T5962 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1493315 T5962 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1493316 T5962 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:20743_zg
   [junit4]   2> 1493316 T5962 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1493317 T5962 oasc.Overseer.start Overseer (id=93617987373367299-127.0.0.1:20743_zg-n_0000000000) starting
   [junit4]   2> 1493318 T5962 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1493322 T5962 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1493323 T5990 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1493323 T5962 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1493323 T5989 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1493326 T5962 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores
   [junit4]   2> 1493328 T5962 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, dataDir=data/, shard=, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores/collection1, loadOnStartup=true, schema=schema.xml, transient=false, collection=control_collection, coreNodeName=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores/collection1/}
   [junit4]   2> 1493329 T5962 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores/collection1/
   [junit4]   2> 1493329 T5962 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1493330 T5992 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1493330 T5992 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1493330 T5988 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1493330 T5992 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1493331 T5989 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20743/zg",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:20743_zg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 1493331 T5989 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20743/zg",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:20743_zg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1493332 T5989 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1493332 T5989 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1493333 T5988 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1494331 T5992 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1494332 T5992 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1494332 T5992 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1494332 T5992 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1494333 T5992 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1494333 T5992 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores/collection1/'
   [junit4]   2> 1494346 T5992 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1494348 T5992 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1494355 T5992 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1494370 T5992 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1494439 T5992 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1494540 T5992 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1494549 T5992 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1494886 T5992 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1494888 T5992 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1494890 T5992 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1494893 T5992 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1494905 T5992 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1494906 T5992 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1494906 T5992 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores/collection1/data/
   [junit4]   2> 1494906 T5992 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4bc15184
   [junit4]   2> 1494907 T5992 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores/collection1/data
   [junit4]   2> 1494907 T5992 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores/collection1/data/index/
   [junit4]   2> 1494907 T5992 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1494907 T5992 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores/collection1/data/index
   [junit4]   2> 1494908 T5992 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=48, maxMergedSegmentMB=96.4521484375, floorSegmentMB=1.4296875, forceMergeDeletesPctAllowed=18.51724424491889, segmentsPerTier=41.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1494908 T5992 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4e06e62a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@26f0f76b),segFN=segments_1,generation=1}
   [junit4]   2> 1494909 T5992 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1494912 T5992 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1494913 T5992 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1494913 T5992 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1494913 T5992 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1494913 T5992 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1494913 T5992 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1494914 T5992 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1494914 T5992 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1494914 T5992 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1494926 T5992 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1494928 T5992 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1494929 T5992 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1494930 T5992 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1494934 T5992 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/json,/admin/plugins,/update/json/docs,/update/csv,/admin/ping,/config,/admin/luke,/admin/system,/admin/threads,/schema,/admin/file,/update,/admin/mbeans,/admin/logging,standard,/admin/segments,/replication,/get,/admin/properties
   [junit4]   2> 1494935 T5992 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1494936 T5992 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1494936 T5992 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1494937 T5992 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1494937 T5992 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1494938 T5992 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=15, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 1494938 T5992 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4e06e62a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@26f0f76b),segFN=segments_1,generation=1}
   [junit4]   2> 1494938 T5992 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1494939 T5992 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@1268aa85[collection1] main
   [junit4]   2> 1494939 T5992 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1494940 T5992 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1494940 T5992 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1494940 T5992 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1494940 T5992 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1494940 T5992 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1494941 T5992 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1494941 T5992 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1494941 T5992 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1494941 T5992 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1494942 T5993 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1268aa85[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1494943 T5992 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1494943 T5996 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:20743/zg collection:control_collection shard:shard1
   [junit4]   2> 1494944 T5962 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 1494944 T5962 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1494944 T5996 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1494946 T5962 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1494947 T5962 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1494948 T5996 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1494948 T5999 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7a1acfd1 name:ZooKeeperConnection Watcher:127.0.0.1:20738/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1494949 T5962 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1494949 T5962 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1494950 T5962 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1494950 T5996 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1494950 T5996 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1494950 T5988 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> ASYNC  NEW_CORE C2726 name=collection1 org.apache.solr.core.SolrCore@46237e29 url=http://127.0.0.1:20743/zg/collection1 node=127.0.0.1:20743_zg C2726_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:20743/zg, node_name=127.0.0.1:20743_zg, core=collection1, state=down}
   [junit4]   2> 1494950 T5996 C:control_collection S:shard1 C2726 P20743 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:20743/zg/collection1/
   [junit4]   2> 1494951 T5996 C:control_collection S:shard1 C2726 P20743 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1494951 T5996 C:control_collection S:shard1 C2726 P20743 oasc.SyncStrategy.syncToMe http://127.0.0.1:20743/zg/collection1/ has no replicas
   [junit4]   2> 1494951 T5989 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 1494951 T5996 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:20743/zg/collection1/ shard1
   [junit4]   2> 1494952 T5962 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1494952 T5996 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1494952 T5962 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 1494952 T5999 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1494952 T5962 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1494952 T5988 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1494954 T5962 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1494956 T6002 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4cf39acf name:ZooKeeperConnection Watcher:127.0.0.1:20738/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1494956 T5962 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1494956 T5962 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1494957 T5988 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1494958 T5989 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20743/zg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 1494960 T5989 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"4",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 1494960 T5989 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 1494961 T5999 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1494961 T5988 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1494963 T5988 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1494963 T5989 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 1494964 T5999 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1494964 T5988 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1495008 T5996 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:20743/zg/collection1/ and leader is http://127.0.0.1:20743/zg/collection1/
   [junit4]   2> 1495009 T5996 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:20743/zg
   [junit4]   2> 1495009 T5996 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1495009 T5996 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1495009 T5996 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1495011 T5988 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1495011 T5989 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20743/zg",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:20743_zg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"4",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state"} current state version: 4
   [junit4]   2> 1495012 T5989 oasco.ReplicaMutator.updateState Update state numShards=4 message={
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20743/zg",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:20743_zg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"4",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1495115 T5988 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1495115 T5999 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1496198 T5962 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores/collection1
   [junit4]   2> 1496200 T5962 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001
   [junit4]   2> 1496201 T5962 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1496203 T5962 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:20767
   [junit4]   2> 1496204 T5962 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=20767, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores, hostContext=/zg, solrconfig=solrconfig.xml}
   [junit4]   2> 1496204 T5962 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 1496205 T5962 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/'
   [junit4]   2> 1496236 T5962 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/solr.xml
   [junit4]   2> 1496273 T5962 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores
   [junit4]   2> 1496274 T5962 oasc.CoreContainer.<init> New CoreContainer 211441626
   [junit4]   2> 1496274 T5962 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/]
   [junit4]   2> 1496274 T5962 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/lib
   [junit4]   2> 1496275 T5962 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/lib).
   [junit4]   2> 1496284 T5962 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1496285 T5962 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1496285 T5962 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1496285 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1496285 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1496286 T5962 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1496286 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1496286 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1496286 T5962 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1496286 T5962 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1496287 T5962 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1496291 T5962 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1496292 T5962 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1496293 T5962 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1496293 T5962 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1496293 T5962 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:20738/solr
   [junit4]   2> 1496293 T5962 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1496293 T5962 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1496294 T5962 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1496296 T6014 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@179bb877 name:ZooKeeperConnection Watcher:127.0.0.1:20738 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1496297 T5962 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1496297 T5962 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1496299 T5962 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1496300 T6017 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@9c04ef6 name:ZooKeeperConnection Watcher:127.0.0.1:20738/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1496300 T5962 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1496303 T5962 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1497307 T5962 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:20767_zg
   [junit4]   2> 1497308 T5962 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:20767_zg
   [junit4]   2> 1497312 T5962 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1497314 T5962 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores
   [junit4]   2> 1497319 T5962 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {schema=schema.xml, loadOnStartup=true, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores/collection1, dataDir=data/, shard=, coreNodeName=, collection=collection1, name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores/collection1/, config=solrconfig.xml}
   [junit4]   2> 1497319 T5962 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores/collection1/
   [junit4]   2> 1497319 T5962 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1497321 T6018 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1497321 T6018 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1497323 T5988 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1497323 T6018 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 1497323 T6018 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 1497324 T5989 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20767/zg",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:20767_zg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"4",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state"} current state version: 5
   [junit4]   2> 1497324 T6018 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 1497324 T5989 oasco.ReplicaMutator.updateState Update state numShards=4 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20767/zg",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:20767_zg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"4",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1497324 T5989 oasco.ReplicaMutator.updateState Collection already exists with numShards=4
   [junit4]   2> 1497325 T5989 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 1497325 T6018 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 1497325 T6018 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1497326 T5989 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 1497326 T6017 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1497327 T6017 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 1498325 T6018 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1498326 T6018 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1498326 T6018 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1498326 T6018 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1498327 T6018 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1498327 T6018 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores/collection1/'
   [junit4]   2> 1498340 T6018 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1498342 T6018 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1498349 T6018 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1498364 T6018 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1498399 T6018 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1498500 T6018 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1498508 T6018 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1498770 T6018 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1498771 T6018 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1498773 T6018 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1498776 T6018 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1498788 T6018 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1498789 T6018 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1498789 T6018 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 1498789 T6018 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4bc15184
   [junit4]   2> 1498790 T6018 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores/collection1/data
   [junit4]   2> 1498791 T6018 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 1498791 T6018 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1498791 T6018 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 1498792 T6018 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=48, maxMergedSegmentMB=96.4521484375, floorSegmentMB=1.4296875, forceMergeDeletesPctAllowed=18.51724424491889, segmentsPerTier=41.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1498793 T6018 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@47337b54 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4e5ec51),segFN=segments_1,generation=1}
   [junit4]   2> 1498793 T6018 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1498800 T6018 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1498800 T6018 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1498801 T6018 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1498801 T6018 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1498801 T6018 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1498802 T6018 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1498802 T6018 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1498802 T6018 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1498803 T6018 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1498822 T6018 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1498825 T6018 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1498827 T6018 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1498830 T6018 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1498837 T6018 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/json,/admin/plugins,/update/json/docs,/update/csv,/admin/ping,/config,/admin/luke,/admin/system,/admin/threads,/schema,/admin/file,/update,/admin/mbeans,/admin/logging,standard,/admin/segments,/replication,/get,/admin/properties
   [junit4]   2> 1498839 T6018 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1498841 T6018 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1498841 T6018 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1498843 T6018 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1498843 T6018 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1498844 T6018 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=15, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 1498845 T6018 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@47337b54 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4e5ec51),segFN=segments_1,generation=1}
   [junit4]   2> 1498845 T6018 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1498845 T6018 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@49167439[collection1] main
   [junit4]   2> 1498846 T6018 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1498846 T6018 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1498846 T6018 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1498847 T6018 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1498847 T6018 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1498847 T6018 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1498847 T6018 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1498848 T6018 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1498848 T6018 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1498848 T6018 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1498849 T6019 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@49167439[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1498849 T6018 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1498850 T6022 C:collection1 S:shard2 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:20767/zg collection:collection1 shard:shard2
   [junit4]   2> 1498850 T5962 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 1498851 T5962 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1498851 T6022 C:collection1 S:shard2 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 1498854 T6022 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 1498855 T5988 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1498855 T6022 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1498856 T6022 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C2727 name=collection1 org.apache.solr.core.SolrCore@2d6de342 url=http://127.0.0.1:20767/zg/collection1 node=127.0.0.1:20767_zg C2727_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:20767/zg, node_name=127.0.0.1:20767_zg, core=collection1, state=down}
   [junit4]   2> 1498856 T6022 C:collection1 S:shard2 C2727 P20767 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:20767/zg/collection1/
   [junit4]   2> 1498856 T5989 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 1498856 T6022 C:collection1 S:shard2 C2727 P20767 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1498857 T6022 C:collection1 S:shard2 C2727 P20767 oasc.SyncStrategy.syncToMe http://127.0.0.1:20767/zg/collection1/ has no replicas
   [junit4]   2> 1498857 T6022 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:20767/zg/collection1/ shard2
   [junit4]   2> 1498857 T6022 C:collection1 S:shard2 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 1498857 T5989 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 1498858 T6017 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1498858 T6017 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 1498860 T5989 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20767/zg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 1498861 T5989 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 1498861 T6017 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1498862 T6017 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 1498863 T5988 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1498910 T6022 C:collection1 S:shard2 oasc.ZkController.register We are http://127.0.0.1:20767/zg/collection1/ and leader is http://127.0.0.1:20767/zg/collection1/
   [junit4]   2> 1498911 T6022 C:collection1 S:shard2 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:20767/zg
   [junit4]   2> 1498911 T6022 C:collection1 S:shard2 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1498911 T6022 C:collection1 S:shard2 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1498911 T6022 C:collection1 S:shard2 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1498912 T5988 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1498913 T5989 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20767/zg",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:20767_zg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"4",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "operation":"state"} current state version: 5
   [junit4]   2> 1498914 T5989 oasco.ReplicaMutator.updateState Update state numShards=4 message={
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20767/zg",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:20767_zg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"4",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1498914 T6022 C:collection1 S:shard2 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 1498914 T5989 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 3
   [junit4]   2> 1498915 T6017 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1498915 T6017 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 1499784 T5962 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-2-001/cores/collection1
   [junit4]   2> 1499786 T5962 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-2-001
   [junit4]   2> 1499787 T5962 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1499790 T5962 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:20781
   [junit4]   2> 1499790 T5962 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-2-001/cores, hostPort=20781, solrconfig=solrconfig.xml, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/tempDir-001/jetty2, hostContext=/zg}
   [junit4]   2> 1499790 T5962 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 1499791 T5962 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-2-001/'
   [junit4]   2> 1499822 T5962 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-2-001/solr.xml
   [junit4]   2> 1499865 T5962 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-2-001/cores
   [junit4]   2> 1499866 T5962 oasc.CoreContainer.<init> New CoreContainer 1551576367
   [junit4]   2> 1499866 T5962 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-2-001/]
   [junit4]   2> 1499866 T5962 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-2-001/lib
   [junit4]   2> 1499867 T5962 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-2-001/lib).
   [junit4]   2> 1499877 T5962 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1499877 T5962 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1499878 T5962 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1499878 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1499878 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1499878 T5962 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1499878 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1499879 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1499879 T5962 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1499879 T5962 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1499879 T5962 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1499885 T5962 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client w

[...truncated too long message...]

kStateReader.java:137)
   [junit4]    > 	at org.apache.solr.common.cloud.ZkNodeProps.load(ZkNodeProps.java:92)
   [junit4]    > 	at org.apache.solr.cloud.DistributedQueue.containsTaskWithRequestId(DistributedQueue.java:127)
   [junit4]    > 	at org.apache.solr.handler.admin.CollectionsHandler.overseerCollectionQueueContains(CollectionsHandler.java:647)
   [junit4]    > 	at org.apache.solr.handler.admin.CollectionsHandler.handleResponse(CollectionsHandler.java:669)
   [junit4]    > 	at org.apache.solr.handler.admin.CollectionsHandler.handleResponse(CollectionsHandler.java:652)
   [junit4]    > 	at org.apache.solr.handler.admin.CollectionsHandler.rejoinElection(CollectionsHandler.java:447)
   [junit4]    > 	at org.apache.solr.handler.admin.CollectionsHandler.insurePreferredIsLeader(CollectionsHandler.java:362)
   [junit4]    > 	at org.apache.solr.handler.admin.CollectionsHandler.handleBalanceLeaders(CollectionsHandler.java:269)
   [junit4]    > 	at org.apache.solr.handler.admin.CollectionsHandler.handleRequestBody(CollectionsHandler.java:235)
   [junit4]    > 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:143)
   [junit4]    > 	at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:782)
   [junit4]    > 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:282)
   [junit4]    > 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
   [junit4]    > 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:102)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
   [junit4]    > 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
   [junit4]    > 	at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
   [junit4]    > 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
   [junit4]    > 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
   [junit4]    > 	at org.eclipse.jetty.server.Server.handle(Server.java:368)
   [junit4]    > 	at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
   [junit4]    > 	at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
   [junit4]    > 	at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
   [junit4]    > 	at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
   [junit4]    > 	at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
   [junit4]    > 	at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
   [junit4]    > 	at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
   [junit4]    > 	at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
   [junit4]    > 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
   [junit4]    > 	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > ,code=500}</pre></p>
   [junit4]    > <hr /><i><small>Powered by Jetty://</small></i>
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    > </body>
   [junit4]    > </html>
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:525)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:338)
   [junit4]    > 	... 48 more
   [junit4]   2> 1592283 T5962 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=it_CH, timezone=Asia/Calcutta
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=211655760,total=402653184
   [junit4]   2> NOTE: All tests run in this JVM: [TestSolrQueryParserDefaultOperatorResource, TestNRTOpen, WordBreakSolrSpellCheckerTest, TestSolrXml, TestUpdate, TestCloudInspectUtil, TestElisionMultitermQuery, SpatialHeatmapFacetsTest, TestFieldCollectionResource, TestDistributedSearch, AnalysisErrorHandlingTest, TestLazyCores, DistribDocExpirationUpdateProcessorTest, CloudMLTQParserTest, DistributedFacetPivotLongTailTest, TestObjectReleaseTracker, TestSolrQueryParserResource, TestDistributedGrouping, FieldAnalysisRequestHandlerTest, ShardSplitTest, TestPostingsSolrHighlighter, DocValuesTest, FieldMutatingUpdateProcessorTest, TestManagedSynonymFilterFactory, PreAnalyzedFieldTest, DistributedQueryElevationComponentTest, TestInfoStreamLogging, HdfsChaosMonkeySafeLeaderTest, DistributedExpandComponentTest, OpenCloseCoreStressTest, BadCopyFieldTest, ClusterStateTest, TestStressLucene, CoreAdminCreateDiscoverTest, DeleteShardTest, HdfsBasicDistributedZkTest, TestSolrDeletionPolicy1, TestReload, ZkSolrClientTest, SpellCheckCollatorTest, AutoCommitTest, SpatialRPTFieldTypeTest, TestSuggestSpellingConverter, TestReplicaProperties, SharedFSAutoReplicaFailoverUtilsTest, CoreMergeIndexesAdminHandlerTest, TestOmitPositions, DirectSolrSpellCheckerTest, TestStressUserVersions, SpatialFilterTest, ZkStateWriterTest, DeleteReplicaTest, HdfsDirectoryTest, BadComponentTest, SortByFunctionTest, TestCustomSort, SolrPluginUtilsTest, TestCloudManagedSchemaConcurrent, TestCloudSchemaless, LeaderInitiatedRecoveryOnCommitTest, ShardRoutingTest, TestRebalanceLeaders]
   [junit4] Completed on J2 in 101.12s, 1 test, 1 error <<< FAILURES!

[...truncated 673 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 480 suites, 1905 tests, 1 error, 62 ignored (22 assumptions)

Total time: 58 minutes 9 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 212847609 bytes
Compression is 0.0%
Took 41 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2910 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2910/

1 tests failed.
REGRESSION:  org.apache.lucene.search.join.TestJoinUtil.testRandomOrdinalsJoin

Error Message:
expected:<296> but was:<269>

Stack Trace:
java.lang.AssertionError: expected:<296> but was:<269>
	at __randomizedtesting.SeedInfo.seed([5214ECCDEBB46393:EDC9ED543B48769F]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.lucene.search.join.TestJoinUtil.assertTopDocs(TestJoinUtil.java:714)
	at org.apache.lucene.search.join.TestJoinUtil.testRandomOrdinalsJoin(TestJoinUtil.java:345)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 6851 lines...]
   [junit4] Suite: org.apache.lucene.search.join.TestJoinUtil
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestJoinUtil -Dtests.method=testRandomOrdinalsJoin -Dtests.seed=5214ECCDEBB46393 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=de_LU -Dtests.timezone=Europe/Jersey -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE 0.76s J1 | TestJoinUtil.testRandomOrdinalsJoin <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: expected:<296> but was:<269>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([5214ECCDEBB46393:EDC9ED543B48769F]:0)
   [junit4]    > 	at org.apache.lucene.search.join.TestJoinUtil.assertTopDocs(TestJoinUtil.java:714)
   [junit4]    > 	at org.apache.lucene.search.join.TestJoinUtil.testRandomOrdinalsJoin(TestJoinUtil.java:345)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {movieId=PostingsFormat(name=MockRandom), name=PostingsFormat(name=MockRandom), id=PostingsFormat(name=MockRandom), description=PostingsFormat(name=Memory doPackFST= true), type=PostingsFormat(name=LuceneVarGapFixedInterval), to=PostingsFormat(name=MockRandom), price=PostingsFormat(name=LuceneFixedGap), value=PostingsFormat(name=LuceneFixedGap), from=PostingsFormat(name=LuceneVarGapFixedInterval), subtitle=PostingsFormat(name=Memory doPackFST= true), productId=PostingsFormat(name=LuceneVarGapFixedInterval)}, docValues:{movieId=DocValuesFormat(name=Asserting), id=DocValuesFormat(name=Asserting), to=DocValuesFormat(name=Asserting), join_field=DocValuesFormat(name=Memory), idproductId=DocValuesFormat(name=Asserting), from=DocValuesFormat(name=Lucene50), productId=DocValuesFormat(name=Lucene50)}, sim=DefaultSimilarity, locale=de_LU, timezone=Europe/Jersey
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=151345944,total=272105472
   [junit4]   2> NOTE: All tests run in this JVM: [TestJoinUtil]
   [junit4] Completed on J1 in 15.11s, 9 tests, 1 failure <<< FAILURES!

[...truncated 12 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/build.xml:456: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:2166: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/module-build.xml:58: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 4 suites, 29 tests, 1 failure

Total time: 18 minutes 42 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 62472 bytes
Compression is 0.0%
Took 0.27 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2909 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2909/

All tests passed

Build Log:
[...truncated 28423 lines...]
check-licenses:
     [echo] License check under: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr
 [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/jcl-over-slf4j-1.7.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/jcl-over-slf4j-1.7.6.jar.sha1

 [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/jul-to-slf4j-1.7.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/jul-to-slf4j-1.7.6.jar.sha1
 [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/slf4j-api-1.7.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/slf4j-api-1.7.6.jar.sha1

[...truncated 3 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:90: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:256: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/tools/custom-tasks.xml:62: License check failed. Check the logs.
If you recently modified ivy-versions.properties or any module's ivy.xml,
make sure you run "ant clean-jars jar-checksums" before running precommit.

Total time: 93 minutes 35 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 1 artifacts
Archive block size is 32768
Received 0 blocks and 464 bytes
Compression is 0.0%
Took 0.2 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2908 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2908/

1 tests failed.
FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
java.net.SocketTimeoutException: Read timed out

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: java.net.SocketTimeoutException: Read timed out
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.sendDocsWithRetry(AbstractFullDistribZkTestBase.java:774)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:277)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 10147 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/init-core-data-001
   [junit4]   2> 1371559 T4624 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /nz_fss/la
   [junit4]   2> 1371564 T4624 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1371565 T4625 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1371666 T4624 oasc.ZkTestServer.run start zk server on port:25779
   [junit4]   2> 1371667 T4624 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1371668 T4624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1371673 T4632 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4e862073 name:ZooKeeperConnection Watcher:127.0.0.1:25779 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1371673 T4624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1371674 T4624 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1371674 T4624 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1371679 T4624 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1371680 T4624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1371681 T4635 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@268cfa3e name:ZooKeeperConnection Watcher:127.0.0.1:25779/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1371681 T4624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1371682 T4624 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1371682 T4624 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1371685 T4624 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1371686 T4624 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1371687 T4624 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1371689 T4624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1371689 T4624 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1371692 T4624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1371693 T4624 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1371796 T4624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1371797 T4624 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1371798 T4624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1371799 T4624 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1371801 T4624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1371801 T4624 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1371803 T4624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1371804 T4624 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1371805 T4624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1371806 T4624 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1371807 T4624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1371808 T4624 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1371810 T4624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1371810 T4624 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1371812 T4624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1371812 T4624 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1371814 T4624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1371815 T4624 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1373224 T4624 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores/collection1
   [junit4]   2> 1373227 T4624 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1373233 T4624 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:25783
   [junit4]   2> 1373233 T4624 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/nz_fss/la, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/tempDir-001/control/data, hostPort=25782, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores}
   [junit4]   2> 1373233 T4624 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 1373234 T4624 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/'
   [junit4]   2> 1373272 T4624 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/solr.xml
   [junit4]   2> 1373316 T4624 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores
   [junit4]   2> 1373316 T4624 oasc.CoreContainer.<init> New CoreContainer 1448812155
   [junit4]   2> 1373316 T4624 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/]
   [junit4]   2> 1373316 T4624 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/lib
   [junit4]   2> 1373317 T4624 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/lib).
   [junit4]   2> 1373326 T4624 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1373327 T4624 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1373327 T4624 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1373327 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1373327 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1373328 T4624 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1373328 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1373328 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1373328 T4624 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1373328 T4624 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1373329 T4624 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1373335 T4624 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1373335 T4624 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1373336 T4624 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1373336 T4624 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1373336 T4624 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:25779/solr
   [junit4]   2> 1373336 T4624 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1373337 T4624 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1373337 T4624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1373339 T4647 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@772c4729 name:ZooKeeperConnection Watcher:127.0.0.1:25779 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1373340 T4624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1373340 T4624 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1373342 T4624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1373343 T4650 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3548fe79 name:ZooKeeperConnection Watcher:127.0.0.1:25779/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1373343 T4624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1373345 T4624 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1373346 T4624 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1373348 T4624 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1373349 T4624 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1373350 T4624 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1373351 T4624 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1373352 T4624 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1373353 T4624 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1373354 T4624 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:25782_nz_fss%2Fla
   [junit4]   2> 1373354 T4624 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:25782_nz_fss%2Fla
   [junit4]   2> 1373355 T4624 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1373356 T4624 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1373357 T4624 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1373358 T4624 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:25782_nz_fss%2Fla
   [junit4]   2> 1373358 T4624 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1373359 T4624 oasc.Overseer.start Overseer (id=93615144876572675-127.0.0.1:25782_nz_fss%2Fla-n_0000000000) starting
   [junit4]   2> 1373361 T4624 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1373364 T4624 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1373365 T4652 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1373365 T4651 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1373365 T4624 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1373369 T4624 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores
   [junit4]   2> 1373371 T4624 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores/collection1/, transient=false, config=solrconfig.xml, coreNodeName=, loadOnStartup=true, shard=, dataDir=data/, collection=control_collection, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores/collection1, schema=schema.xml, name=collection1}
   [junit4]   2> 1373371 T4624 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores/collection1/
   [junit4]   2> 1373371 T4624 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1373372 T4654 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1373372 T4654 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1373373 T4654 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1373373 T4650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1373373 T4651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:25782_nz_fss%2Fla",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25782/nz_fss/la",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 1373374 T4651 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:25782_nz_fss%2Fla",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25782/nz_fss/la",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1373374 T4651 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1373374 T4651 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1373375 T4650 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1374374 T4654 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1374375 T4654 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1374375 T4654 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1374375 T4654 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1374376 T4654 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1374376 T4654 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores/collection1/'
   [junit4]   2> 1374390 T4654 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1374392 T4654 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1374400 T4654 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1374419 T4654 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1374457 T4654 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1374559 T4654 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1374567 T4654 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1374890 T4654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1374903 T4654 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1374905 T4654 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1374912 T4654 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1374915 T4654 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1374919 T4654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1374919 T4654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1374919 T4654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1374919 T4654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1374920 T4654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1374920 T4654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1374920 T4654 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1374920 T4654 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1374921 T4654 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores/collection1/data/
   [junit4]   2> 1374921 T4654 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6ba9dace
   [junit4]   2> 1374921 T4654 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores/collection1/data
   [junit4]   2> 1374922 T4654 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores/collection1/data/index/
   [junit4]   2> 1374922 T4654 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1374922 T4654 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores/collection1/data/index
   [junit4]   2> 1374922 T4654 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=46, maxMergedSegmentMB=23.0185546875, floorSegmentMB=1.162109375, forceMergeDeletesPctAllowed=11.77904699044102, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1374923 T4654 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@763fe6fe lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5d566a58),segFN=segments_1,generation=1}
   [junit4]   2> 1374923 T4654 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1374927 T4654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1374927 T4654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1374927 T4654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1374928 T4654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1374928 T4654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1374928 T4654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1374928 T4654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1374928 T4654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1374929 T4654 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1374951 T4654 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1374953 T4654 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1374954 T4654 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1374956 T4654 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1374960 T4654 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/plugins,/admin/luke,/admin/segments,/admin/file,/admin/threads,/admin/ping,/update,/admin/mbeans,/admin/logging,/admin/properties,/update/csv,/config,/replication,/get,/schema,standard,/update/json,/admin/system,/update/json/docs
   [junit4]   2> 1374962 T4654 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1374963 T4654 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1374963 T4654 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1374965 T4654 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1374965 T4654 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1374965 T4654 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=43, maxMergeAtOnceExplicit=46, maxMergedSegmentMB=69.14453125, floorSegmentMB=1.6064453125, forceMergeDeletesPctAllowed=4.444172208952494, segmentsPerTier=45.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4601051453371784
   [junit4]   2> 1374966 T4654 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@763fe6fe lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5d566a58),segFN=segments_1,generation=1}
   [junit4]   2> 1374966 T4654 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1374966 T4654 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@1ea1aa6a[collection1] main
   [junit4]   2> 1374966 T4654 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1374967 T4654 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1374967 T4654 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1374967 T4654 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1374968 T4654 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1374968 T4654 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1374968 T4654 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1374968 T4654 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1374968 T4654 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1374969 T4654 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1374969 T4654 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1374969 T4655 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1ea1aa6a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1374970 T4658 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:25782/nz_fss/la collection:control_collection shard:shard1
   [junit4]   2> 1374970 T4624 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 1374970 T4624 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1374970 T4658 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1374972 T4624 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1374973 T4624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1374974 T4658 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1374974 T4662 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5aedef0c name:ZooKeeperConnection Watcher:127.0.0.1:25779/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1374974 T4624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1374975 T4624 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1374975 T4624 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1374975 T4650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1374976 T4658 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1374976 T4658 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C871 name=collection1 org.apache.solr.core.SolrCore@7aad97f2 url=http://127.0.0.1:25782/nz_fss/la/collection1 node=127.0.0.1:25782_nz_fss%2Fla C871_STATE=coll:control_collection core:collection1 props:{core=collection1, node_name=127.0.0.1:25782_nz_fss%2Fla, base_url=http://127.0.0.1:25782/nz_fss/la, state=down}
   [junit4]   2> 1374977 T4658 C:control_collection S:shard1 C871 P25782 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:25782/nz_fss/la/collection1/
   [junit4]   2> 1374977 T4651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 1374977 T4658 C:control_collection S:shard1 C871 P25782 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1374977 T4658 C:control_collection S:shard1 C871 P25782 oasc.SyncStrategy.syncToMe http://127.0.0.1:25782/nz_fss/la/collection1/ has no replicas
   [junit4]   2> 1374977 T4658 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:25782/nz_fss/la/collection1/ shard1
   [junit4]   2> 1374977 T4658 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1374978 T4650 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1374978 T4662 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1374978 T4624 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1374979 T4624 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 1374980 T4624 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1374980 T4624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1374981 T4650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1374982 T4665 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1cde29fe name:ZooKeeperConnection Watcher:127.0.0.1:25779/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1374982 T4651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25782/nz_fss/la",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 1374982 T4624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1374982 T4624 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1374985 T4650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1374986 T4651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 1374986 T4651 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 1374987 T4662 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1374987 T4650 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1375032 T4658 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:25782/nz_fss/la/collection1/ and leader is http://127.0.0.1:25782/nz_fss/la/collection1/
   [junit4]   2> 1375033 T4658 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:25782/nz_fss/la
   [junit4]   2> 1375033 T4658 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1375033 T4658 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1375033 T4658 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1375034 T4650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1375034 T4651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:25782_nz_fss%2Fla",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25782/nz_fss/la",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state"} current state version: 3
   [junit4]   2> 1375035 T4651 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:25782_nz_fss%2Fla",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25782/nz_fss/la",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1375035 T4651 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 1375036 T4650 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1375036 T4662 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1375139 T4662 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1375139 T4650 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1376129 T4624 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores/collection1
   [junit4]   2> 1376131 T4624 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001
   [junit4]   2> 1376132 T4624 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1376134 T4624 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:25795
   [junit4]   2> 1376135 T4624 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/nz_fss/la, solrconfig=solrconfig.xml, hostPort=25794, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/tempDir-001/jetty1}
   [junit4]   2> 1376135 T4624 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 1376135 T4624 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/'
   [junit4]   2> 1376167 T4624 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/solr.xml
   [junit4]   2> 1376203 T4624 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores
   [junit4]   2> 1376203 T4624 oasc.CoreContainer.<init> New CoreContainer 1379001132
   [junit4]   2> 1376203 T4624 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/]
   [junit4]   2> 1376204 T4624 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/lib
   [junit4]   2> 1376204 T4624 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/lib).
   [junit4]   2> 1376214 T4624 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1376214 T4624 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1376214 T4624 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1376215 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1376215 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1376215 T4624 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1376215 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1376215 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1376216 T4624 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1376216 T4624 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1376216 T4624 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1376221 T4624 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1376222 T4624 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1376222 T4624 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1376222 T4624 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1376222 T4624 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:25779/solr
   [junit4]   2> 1376222 T4624 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1376223 T4624 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1376223 T4624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1376225 T4677 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5ded99f0 name:ZooKeeperConnection Watcher:127.0.0.1:25779 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1376226 T4624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1376226 T4624 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1376229 T4624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1376230 T4680 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@48297555 name:ZooKeeperConnection Watcher:127.0.0.1:25779/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1376230 T4624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1376243 T4624 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1377247 T4624 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:25794_nz_fss%2Fla
   [junit4]   2> 1377248 T4624 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:25794_nz_fss%2Fla
   [junit4]   2> 1377251 T4624 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1377253 T4624 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores
   [junit4]   2> 1377255 T4624 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {schema=schema.xml, dataDir=data/, config=solrconfig.xml, name=collection1, collection=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores/collection1/, loadOnStartup=true, coreNodeName=, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores/collection1, shard=}
   [junit4]   2> 1377255 T4624 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores/collection1/
   [junit4]   2> 1377255 T4624 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1377257 T4681 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1377257 T4681 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1377258 T4650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1377258 T4681 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 1377259 T4681 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 1377259 T4651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:25794_nz_fss%2Fla",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25794/nz_fss/la",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state"} current state version: 5
   [junit4]   2> 1377259 T4681 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 1377259 T4651 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:25794_nz_fss%2Fla",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25794/nz_fss/la",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1377259 T4651 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 1377259 T4651 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1377260 T4681 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 1377260 T4681 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1377260 T4651 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 1377261 T4680 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1377261 T4680 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 1378261 T4681 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1378262 T4681 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1378262 T4681 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1378262 T4681 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1378263 T4681 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1378263 T4681 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores/collection1/'
   [junit4]   2> 1378277 T4681 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1378279 T4681 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1378288 T4681 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1378306 T4681 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1378344 T4681 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1378446 T4681 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1378454 T4681 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1378741 T4681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1378755 T4681 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1378757 T4681 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1378763 T4681 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1378766 T4681 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1378769 T4681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1378770 T4681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1378770 T4681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1378770 T4681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1378771 T4681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1378771 T4681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1378771 T4681 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1378771 T4681 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1378771 T4681 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 1378772 T4681 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6ba9dace
   [junit4]   2> 1378772 T4681 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores/collection1/data
   [junit4]   2> 1378772 T4681 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 1378773 T4681 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1378773 T4681 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 1378773 T4681 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=46, maxMergedSegmentMB=23.0185546875, floorSegmentMB=1.162109375, forceMergeDeletesPctAllowed=11.77904699044102, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1378774 T4681 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@56774af0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@31d8fee9),segFN=segments_1,generation=1}
   [junit4]   2> 1378774 T4681 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1378778 T4681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1378779 T4681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1378779 T4681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1378779 T4681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1378779 T4681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1378779 T4681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1378780 T4681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1378780 T4681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1378780 T4681 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1378797 T4681 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1378799 T4681 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1378801 T4681 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1378802 T4681 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1378807 T4681 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/plugins,/admin/luke,/admin/segments,/admin/file,/admin/threads,/admin/ping,/update,/admin/mbeans,/admin/logging,/admin/properties,/update/csv,/config,/replication,/get,/schema,standard,/update/json,/admin/system,/update/json/docs
   [junit4]   2> 1378809 T4681 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1378810 T4681 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1378811 T4681 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1378812 T4681 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1378812 T4681 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1378813 T4681 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=43, maxMergeAtOnceExplicit=46, maxMergedSegmentMB=69.14453125, floorSegmentMB=1.6064453125, forceMergeDeletesPctAllowed=4.444172208952494, segmentsPerTier=45.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4601051453371784
   [junit4]   2> 1378813 T4681 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@56774af0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@31d8fee9),segFN=segments_1,generation=1}
   [junit4]   2> 1378813 T4681 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1378814 T4681 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@7a6a994e[collection1] main
   [junit4]   2> 1378814 T4681 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1378814 T4681 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1378814 T4681 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1378815 T4681 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1378815 T4681 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1378815 T4681 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1378815 T4681 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1378816 T4681 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1378816 T4681 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1378816 T4681 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1378817 T4682 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7a6a994e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1378817 T4681 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1378818 T4685 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:25794/nz_fss/la collection:collection1 shard:shard1
   [junit4]   2> 1378818 T4624 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 1378818 T4624 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1378818 T4685 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1378821 T4685 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1378822 T4650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1378823 T4685 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1378823 T4685 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C872 name=collection1 org.apache.solr.core.SolrCore@1bffbe5b url=http://127.0.0.1:25794/nz_fss/la/collection1 node=127.0.0.1:25794_nz_fss%2Fla C872_STATE=coll:collection1 core:collection1 props:{core=collection1, node_name=127.0.0.1:25794_nz_fss%2Fla, base_url=http://127.0.0.1:25794/nz_fss/la, state=down}
   [junit4]   2> 1378823 T4685 C:collection1 S:shard1 C872 P25794 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:25794/nz_fss/la/collection1/
   [junit4]   2> 1378823 T4651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 1378823 T4685 C:collection1 S:shard1 C872 P25794 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1378824 T4685 C:collection1 S:shard1 C872 P25794 oasc.SyncStrategy.syncToMe http://127.0.0.1:25794/nz_fss/la/collection1/ has no replicas
   [junit4]   2> 1378824 T4685 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:25794/nz_fss/la/collection1/ shard1
   [junit4]   2> 1378824 T4685 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1378824 T4651 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 1378825 T4680 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1378825 T4680 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 1378827 T4651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25794/nz_fss/la",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 1378827 T4651 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 1378828 T4680 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1378828 T4680 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 1378829 T4650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1378878 T4685 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:25794/nz_fss/la/collection1/ and leader is http://127.0.0.1:25794/nz_fss/la/collection1/
   [junit4]   2> 1378879 T4685 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:25794/nz_fss/la
   [junit4]   2> 1378879 T4685 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1378879 T4685 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1378879 T4685 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1378880 T4650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1378880 T4651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:25794_nz_fss%2Fla",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25794/nz_fss/la",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state"} current state version: 5
   [junit4]   2> 1378881 T4651 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:25794_nz_fss%2Fla",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25794/nz_fss/la",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1378881 T4685 C:collection1 S:shard1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 1378882 T4651 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 3
   [junit4]   2> 1378882 T4680 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1378883 T4680 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 1379746 T4624 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/cores/collection1
   [junit4]   2> 1379748 T4624 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001
   [junit4]   2> 1379749 T4624 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1379752 T4624 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:25804
   [junit4]   2> 1379752 T4624 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/cores, hostPort=25803, hostContext=/nz_fss/la}
   [junit4]   2> 1379753 T4624 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 1379753 T4624 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/'
   [junit4]   2> 1379782 T4624 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/solr.xml
   [junit4]   2> 1379816 T4624 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/cores
   [junit4]   2> 1379817 T4624 oasc.CoreContainer.<init> New CoreContainer 1199359807
   [junit4]   2> 1379817 T4624 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/]
   [junit4]   2> 1379817 T4624 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/lib
   [junit4]   2> 1379818 T4624 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/lib).
   [junit4]   2> 1379827 T4624 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1379827 T4624 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1379827 T4624 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1379827 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1379828 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1379828 T4624 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1379828 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1379828 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1379828 T4624 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1379829 T4624 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1379829 T4624 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1379833 T4624 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1379834 T4624 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1379834 T4624 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1379834 T4624 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1379835 T4624 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:25779/solr
   [junit4]   2> 1379835 T4624 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1379835 T4624 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1379836 T4624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1379837 T4698 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3ca4f3b4 name:ZooKeeperConnection Watcher:127.0.0.1:25779 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1379838 T4624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1379838 T4624 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1379840 T4624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1379841 T4701 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4e7c6687 name:ZooKeeperConnection Watcher:127.0.0.1:25779/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1379841 T4624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1379845 T4624 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1380848 T4624 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:25803_nz_fss%2Fla
   [junit4]   2> 1380849 T4624 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:25803_nz_fss%2Fla
   [junit4]   2> 1380852 T4624 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1380855 T4624 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/cores
   [junit4]   2> 1380857 T4624 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {coreNodeName=, collection=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/cores/collection1/, dataDir=data/, name=collection1, shard=, schema=schema.xml, transient=false, loadOnStartup=true, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/cores/collection1, config=solrconfig.xml}
   [junit4]   2> 1380857 T4624 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/cores/collection1/
   [junit4]   2> 1380858 T4624 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1380859 T4702 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1380859 T4702 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1380860 T4650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1380861 T4702 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 1380861 T4702 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 1380861 T4651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:25803_nz_fss%2Fla",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25803/nz_fss/la",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state"} current state version: 5
   [junit4]   2> 1380861 T4702 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 1380862 T4651 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:25803_nz_fss%2Fla",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25803/nz_fss/la",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1380862 T4651 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 1380862 T4651 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 1380862 T4702 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 1380862 T4702 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1380863 T4651 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 4
   [junit4]   2> 1380863 T4701 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1380863 T4680 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1380864 T4701 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 5 
   [junit4]   2> 1380864 T4680 oascc.ZkStateReader.update

[...truncated too long message...]

.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:278)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #4: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=4829, name=updateExecutor-653-thread-6, state=RUNNABLE, group=TGRP-ReplicationFactorTest]
   [junit4]    > Caused by: java.util.concurrent.RejectedExecutionException: Task org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner@282848d9 rejected from java.util.concurrent.ThreadPoolExecutor@e71bb73[Shutting down, pool size = 4, active threads = 4, queued tasks = 0, completed tasks = 46]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([BFFCE504F86C54DD]:0)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:278)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #5: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=4832, name=updateExecutor-653-thread-7, state=RUNNABLE, group=TGRP-ReplicationFactorTest]
   [junit4]    > Caused by: java.util.concurrent.RejectedExecutionException: Task org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner@5b761ce3 rejected from java.util.concurrent.ThreadPoolExecutor@e71bb73[Shutting down, pool size = 3, active threads = 3, queued tasks = 0, completed tasks = 47]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([BFFCE504F86C54DD]:0)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:278)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #6: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=4835, name=updateExecutor-653-thread-8, state=RUNNABLE, group=TGRP-ReplicationFactorTest]
   [junit4]    > Caused by: java.util.concurrent.RejectedExecutionException: Task org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner@f2d440b rejected from java.util.concurrent.ThreadPoolExecutor@e71bb73[Shutting down, pool size = 2, active threads = 2, queued tasks = 0, completed tasks = 48]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([BFFCE504F86C54DD]:0)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:278)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #7: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=4822, name=updateExecutor-653-thread-4, state=RUNNABLE, group=TGRP-ReplicationFactorTest]
   [junit4]    > Caused by: java.util.concurrent.RejectedExecutionException: Task org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner@ad15bd2 rejected from java.util.concurrent.ThreadPoolExecutor@e71bb73[Shutting down, pool size = 1, active threads = 1, queued tasks = 0, completed tasks = 49]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([BFFCE504F86C54DD]:0)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:278)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1955257 T4624 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {_version_=PostingsFormat(name=Asserting), timestamp=PostingsFormat(name=LuceneFixedGap), id=PostingsFormat(name=Memory doPackFST= true), a_t=PostingsFormat(name=LuceneFixedGap), intDefault=PostingsFormat(name=Asserting), range_facet_l=PostingsFormat(name=LuceneFixedGap), text=PostingsFormat(name=LuceneVarGapDocFreqInterval), multiDefault=PostingsFormat(name=LuceneFixedGap)}, docValues:{timestamp=DocValuesFormat(name=Asserting)}, sim=DefaultSimilarity, locale=ar_OM, timezone=Europe/Tiraspol
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=78560712,total=365953024
   [junit4]   2> NOTE: All tests run in this JVM: [TestSystemIdResolver, TestSchemaSimilarityResource, PolyFieldTest, CoreAdminCreateDiscoverTest, PrimitiveFieldTypeTest, RegexBoostProcessorTest, PreAnalyzedUpdateProcessorTest, HttpPartitionTest, PreAnalyzedFieldTest, TestSolrQueryParserResource, IndexSchemaRuntimeFieldTest, TestArbitraryIndexDir, TestReload, NumericFieldsTest, TestJmxMonitoredMap, TestBulkSchemaAPI, TestManagedResource, TestSolrQueryParserDefaultOperatorResource, TestFieldTypeResource, AnalysisAfterCoreReloadTest, DocValuesMissingTest, DistributedMLTComponentTest, EchoParamsTest, TermVectorComponentDistributedTest, TestCloudInspectUtil, QueryResultKeyTest, TestDistributedGrouping, HdfsBasicDistributedZkTest, AddSchemaFieldsUpdateProcessorFactoryTest, TestCollationField, TestLRUCache, TestSolrDeletionPolicy1, TestExtendedDismaxParser, TestTrackingShardHandlerFactory, SolrInfoMBeanTest, BlockDirectoryTest, XsltUpdateRequestHandlerTest, UnloadDistributedZkTest, TestBinaryField, DirectUpdateHandlerTest, CollectionReloadTest, SimpleMLTQParserTest, TestUniqueKeyFieldResource, TestDynamicLoading, TestLeaderElectionZkExpiry, TestManagedSchemaDynamicFieldResource, TestAddFieldRealTimeGet, TestDistribDocBasedVersion, DistributedIntervalFacetingTest, TestJsonFacets, UpdateParamsTest, HdfsSyncSliceTest, TestFastLRUCache, IndexSchemaTest, FieldAnalysisRequestHandlerTest, TestManagedSchemaFieldTypeResource, TestConfig, TestSolrJ, CollectionsAPIDistributedZkTest, BasicDistributedZkTest, MultiThreadedOCPTest, ReplicationFactorTest]
   [junit4] Completed on J0 in 583.73s, 1 test, 1 error <<< FAILURES!

[...truncated 614 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 480 suites, 1905 tests, 1 error, 61 ignored (21 assumptions)

Total time: 60 minutes 33 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 218537067 bytes
Compression is 0.0%
Took 41 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2907 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2907/

1 tests failed.
FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
java.net.SocketTimeoutException: Read timed out

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: java.net.SocketTimeoutException: Read timed out
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.sendDocsWithRetry(AbstractFullDistribZkTestBase.java:774)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:277)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9900 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/init-core-data-001
   [junit4]   2> 440254 T1875 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 440258 T1875 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 440258 T1876 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 440358 T1875 oasc.ZkTestServer.run start zk server on port:14242
   [junit4]   2> 440359 T1875 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 440360 T1875 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 440362 T1883 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@29b8b880 name:ZooKeeperConnection Watcher:127.0.0.1:14242 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 440363 T1875 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 440363 T1875 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 440363 T1875 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 440365 T1875 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 440366 T1875 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 440367 T1886 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3bb633cd name:ZooKeeperConnection Watcher:127.0.0.1:14242/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 440367 T1875 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 440367 T1875 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 440367 T1875 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 440368 T1875 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 440369 T1875 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 440370 T1875 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 440372 T1875 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 440372 T1875 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 440374 T1875 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 440374 T1875 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 440475 T1875 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 440476 T1875 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 440477 T1875 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 440477 T1875 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 440479 T1875 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 440479 T1875 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 440480 T1875 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 440481 T1875 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 440482 T1875 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 440482 T1875 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 440483 T1875 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 440484 T1875 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 440485 T1875 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 440485 T1875 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 440486 T1875 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 440487 T1875 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 440488 T1875 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 440488 T1875 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 441510 T1875 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores/collection1
   [junit4]   2> 441513 T1875 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 441517 T1875 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:14246
   [junit4]   2> 441517 T1875 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=14245, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores}
   [junit4]   2> 441518 T1875 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 441518 T1875 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/'
   [junit4]   2> 441547 T1875 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/solr.xml
   [junit4]   2> 441599 T1875 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores
   [junit4]   2> 441600 T1875 oasc.CoreContainer.<init> New CoreContainer 1380014511
   [junit4]   2> 441600 T1875 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/]
   [junit4]   2> 441601 T1875 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/lib
   [junit4]   2> 441602 T1875 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/lib).
   [junit4]   2> 441611 T1875 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 441611 T1875 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 441612 T1875 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 441612 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 441612 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 441612 T1875 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 441612 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 441613 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 441613 T1875 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 441613 T1875 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 441613 T1875 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 441618 T1875 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 441619 T1875 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 441619 T1875 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 441619 T1875 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 441619 T1875 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:14242/solr
   [junit4]   2> 441619 T1875 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 441620 T1875 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 441620 T1875 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 441622 T1898 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@994d153 name:ZooKeeperConnection Watcher:127.0.0.1:14242 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 441623 T1875 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 441623 T1875 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 441624 T1875 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 441625 T1901 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5fcb98d6 name:ZooKeeperConnection Watcher:127.0.0.1:14242/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 441626 T1875 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 441627 T1875 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 441628 T1875 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 441629 T1875 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 441631 T1875 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 441632 T1875 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 441633 T1875 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 441634 T1875 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 441635 T1875 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 441636 T1875 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:14245_
   [junit4]   2> 441636 T1875 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:14245_
   [junit4]   2> 441637 T1875 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 441638 T1875 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 441639 T1875 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 441640 T1875 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:14245_
   [junit4]   2> 441640 T1875 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 441641 T1875 oasc.Overseer.start Overseer (id=93614406352961539-127.0.0.1:14245_-n_0000000000) starting
   [junit4]   2> 441642 T1875 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 441646 T1875 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 441647 T1903 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 441647 T1902 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 441647 T1875 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 441650 T1875 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores
   [junit4]   2> 441651 T1875 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores/collection1, loadOnStartup=true, dataDir=data/, transient=false, coreNodeName=, schema=schema.xml, config=solrconfig.xml, collection=control_collection, name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores/collection1/}
   [junit4]   2> 441652 T1875 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores/collection1/
   [junit4]   2> 441652 T1875 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 441653 T1905 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 441653 T1905 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 441653 T1905 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 441653 T1901 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 441654 T1902 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14245",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:14245_",
   [junit4]   2> 	  "shard":null} current state version: 0
   [junit4]   2> 441655 T1902 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14245",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:14245_",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 441655 T1902 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 441655 T1902 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 441656 T1901 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 442654 T1905 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 442655 T1905 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 442655 T1905 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 442656 T1905 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 442656 T1905 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 442656 T1905 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores/collection1/'
   [junit4]   2> 442674 T1905 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 442676 T1905 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 442686 T1905 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 442709 T1905 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 442761 T1905 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 442862 T1905 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 442873 T1905 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 443284 T1905 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 443301 T1905 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 443304 T1905 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 443314 T1905 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 443318 T1905 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 443322 T1905 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 443323 T1905 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 443323 T1905 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 443323 T1905 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 443324 T1905 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 443324 T1905 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 443325 T1905 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 443325 T1905 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 443325 T1905 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores/collection1/data/
   [junit4]   2> 443326 T1905 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@42d7ecc1
   [junit4]   2> 443326 T1905 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores/collection1/data
   [junit4]   2> 443327 T1905 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores/collection1/data/index/
   [junit4]   2> 443327 T1905 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 443327 T1905 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores/collection1/data/index
   [junit4]   2> 443328 T1905 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=42, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6542973963737145]
   [junit4]   2> 443329 T1905 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@45455d46 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4ceec231),segFN=segments_1,generation=1}
   [junit4]   2> 443329 T1905 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 443334 T1905 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 443335 T1905 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 443335 T1905 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 443335 T1905 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 443336 T1905 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 443336 T1905 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 443336 T1905 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 443337 T1905 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 443337 T1905 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 443357 T1905 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 443360 T1905 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 443362 T1905 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 443364 T1905 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 443369 T1905 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/file,/admin/ping,/admin/luke,/admin/threads,/admin/properties,/admin/segments,/get,/admin/mbeans,/schema,/update/json/docs,/config,/update/csv,/admin/logging,/admin/plugins,/update,/update/json,/replication,/admin/system,standard
   [junit4]   2> 443371 T1905 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 443373 T1905 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 443373 T1905 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 443374 T1905 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 443375 T1905 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 443375 T1905 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=37, maxMergedSegmentMB=57.2275390625, floorSegmentMB=0.2529296875, forceMergeDeletesPctAllowed=20.90666064053401, segmentsPerTier=22.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 443376 T1905 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@45455d46 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4ceec231),segFN=segments_1,generation=1}
   [junit4]   2> 443376 T1905 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 443377 T1905 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@73d0e76e[collection1] main
   [junit4]   2> 443377 T1905 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 443377 T1905 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 443378 T1905 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 443378 T1905 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 443378 T1905 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 443378 T1905 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 443379 T1905 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 443379 T1905 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 443379 T1905 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 443380 T1905 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 443380 T1905 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 443381 T1906 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@73d0e76e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 443381 T1909 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:14245 collection:control_collection shard:shard1
   [junit4]   2> 443381 T1875 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 443382 T1875 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 443382 T1909 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 443385 T1909 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 443386 T1901 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 443386 T1909 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 443386 T1909 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 443386 T1875 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 443386 T1902 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C206 name=collection1 org.apache.solr.core.SolrCore@1ab3596b url=http://127.0.0.1:14245/collection1 node=127.0.0.1:14245_ C206_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:14245, core=collection1, node_name=127.0.0.1:14245_}
   [junit4]   2> 443386 T1909 C:control_collection S:shard1 C206 P14245 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:14245/collection1/
   [junit4]   2> 443387 T1875 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 443387 T1909 C:control_collection S:shard1 C206 P14245 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 443388 T1909 C:control_collection S:shard1 C206 P14245 oasc.SyncStrategy.syncToMe http://127.0.0.1:14245/collection1/ has no replicas
   [junit4]   2> 443388 T1901 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 443388 T1909 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:14245/collection1/ shard1
   [junit4]   2> 443388 T1909 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 443389 T1913 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7f02e185 name:ZooKeeperConnection Watcher:127.0.0.1:14242/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 443389 T1875 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 443389 T1875 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 443390 T1875 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 443391 T1901 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 443392 T1902 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14245",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 443392 T1875 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 443494 T1913 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 443494 T1901 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 443544 T1909 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:14245/collection1/ and leader is http://127.0.0.1:14245/collection1/
   [junit4]   2> 443545 T1909 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:14245
   [junit4]   2> 443545 T1909 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 443545 T1909 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 443545 T1909 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 443546 T1901 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 443546 T1902 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14245",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:14245_",
   [junit4]   2> 	  "shard":"shard1"} current state version: 3
   [junit4]   2> 443547 T1902 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14245",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:14245_",
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 443650 T1901 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 443650 T1913 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 444808 T1875 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores/collection1
   [junit4]   2> 444811 T1875 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001
   [junit4]   2> 444812 T1875 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 444815 T1875 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:14251
   [junit4]   2> 444815 T1875 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostPort=14250}
   [junit4]   2> 444816 T1875 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 444816 T1875 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/'
   [junit4]   2> 444856 T1875 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/solr.xml
   [junit4]   2> 444896 T1875 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores
   [junit4]   2> 444896 T1875 oasc.CoreContainer.<init> New CoreContainer 1555043384
   [junit4]   2> 444897 T1875 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/]
   [junit4]   2> 444897 T1875 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/lib
   [junit4]   2> 444897 T1875 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/lib).
   [junit4]   2> 444906 T1875 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 444907 T1875 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 444907 T1875 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 444907 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 444907 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 444908 T1875 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 444908 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 444908 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 444908 T1875 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 444908 T1875 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 444909 T1875 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 444914 T1875 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 444915 T1875 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 444915 T1875 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 444915 T1875 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 444915 T1875 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:14242/solr
   [junit4]   2> 444916 T1875 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 444916 T1875 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 444917 T1875 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 444918 T1925 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3fa3586c name:ZooKeeperConnection Watcher:127.0.0.1:14242 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 444919 T1875 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 444920 T1875 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 444921 T1875 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 444922 T1928 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@128b39c7 name:ZooKeeperConnection Watcher:127.0.0.1:14242/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 444922 T1875 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 444925 T1875 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 445928 T1875 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:14250_
   [junit4]   2> 445929 T1875 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:14250_
   [junit4]   2> 445932 T1875 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 445934 T1875 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores
   [junit4]   2> 445935 T1875 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {transient=false, shard=, coreNodeName=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores/collection1/, config=solrconfig.xml, dataDir=data/, collection=collection1, loadOnStartup=true, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores/collection1, name=collection1, schema=schema.xml}
   [junit4]   2> 445936 T1875 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores/collection1/
   [junit4]   2> 445936 T1875 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 445937 T1929 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 445937 T1929 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 445937 T1929 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 445937 T1901 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 445938 T1902 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14250",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:14250_",
   [junit4]   2> 	  "shard":null} current state version: 4
   [junit4]   2> 445938 T1902 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14250",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:14250_",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 445939 T1902 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 445939 T1902 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 445940 T1901 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 445940 T1928 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 445940 T1913 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 446938 T1929 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 446939 T1929 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 446939 T1929 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 446939 T1929 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 446940 T1929 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 446940 T1929 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores/collection1/'
   [junit4]   2> 446954 T1929 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 446956 T1929 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 446965 T1929 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 446982 T1929 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 447020 T1929 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 447121 T1929 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 447129 T1929 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 447470 T1929 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 447482 T1929 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 447484 T1929 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 447492 T1929 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 447495 T1929 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 447498 T1929 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 447498 T1929 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 447498 T1929 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 447498 T1929 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 447499 T1929 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 447499 T1929 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 447499 T1929 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 447499 T1929 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 447500 T1929 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 447500 T1929 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@42d7ecc1
   [junit4]   2> 447501 T1929 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores/collection1/data
   [junit4]   2> 447501 T1929 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 447501 T1929 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 447501 T1929 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 447501 T1929 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=42, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6542973963737145]
   [junit4]   2> 447502 T1929 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@a1b712e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@fa38831),segFN=segments_1,generation=1}
   [junit4]   2> 447502 T1929 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 447507 T1929 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 447507 T1929 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 447507 T1929 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 447508 T1929 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 447508 T1929 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 447508 T1929 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 447508 T1929 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 447509 T1929 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 447509 T1929 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 447523 T1929 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 447524 T1929 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 447526 T1929 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 447528 T1929 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 447533 T1929 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/file,/admin/ping,/admin/luke,/admin/threads,/admin/properties,/admin/segments,/get,/admin/mbeans,/schema,/update/json/docs,/config,/update/csv,/admin/logging,/admin/plugins,/update,/update/json,/replication,/admin/system,standard
   [junit4]   2> 447535 T1929 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 447536 T1929 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 447536 T1929 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 447537 T1929 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 447537 T1929 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 447539 T1929 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=37, maxMergedSegmentMB=57.2275390625, floorSegmentMB=0.2529296875, forceMergeDeletesPctAllowed=20.90666064053401, segmentsPerTier=22.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 447539 T1929 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@a1b712e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@fa38831),segFN=segments_1,generation=1}
   [junit4]   2> 447539 T1929 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 447540 T1929 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@57b843f7[collection1] main
   [junit4]   2> 447540 T1929 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 447540 T1929 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 447541 T1929 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 447541 T1929 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 447541 T1929 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 447541 T1929 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 447542 T1929 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 447542 T1929 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 447542 T1929 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 447542 T1929 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 447543 T1930 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@57b843f7[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 447543 T1929 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 447544 T1933 C:collection1 S:shard3 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:14250 collection:collection1 shard:shard3
   [junit4]   2> 447544 T1875 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 447544 T1875 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 447545 T1933 C:collection1 S:shard3 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
   [junit4]   2> 447548 T1933 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard3
   [junit4]   2> 447549 T1901 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 447549 T1933 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 447549 T1933 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C207 name=collection1 org.apache.solr.core.SolrCore@ab3b34e url=http://127.0.0.1:14250/collection1 node=127.0.0.1:14250_ C207_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:14250, core=collection1, node_name=127.0.0.1:14250_}
   [junit4]   2> 447549 T1933 C:collection1 S:shard3 C207 P14250 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:14250/collection1/
   [junit4]   2> 447549 T1902 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 447549 T1933 C:collection1 S:shard3 C207 P14250 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 447549 T1933 C:collection1 S:shard3 C207 P14250 oasc.SyncStrategy.syncToMe http://127.0.0.1:14250/collection1/ has no replicas
   [junit4]   2> 447550 T1933 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:14250/collection1/ shard3
   [junit4]   2> 447550 T1933 C:collection1 S:shard3 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard3
   [junit4]   2> 447550 T1901 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 447550 T1928 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 447550 T1913 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 447553 T1901 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 447553 T1902 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14250",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 447655 T1901 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 447655 T1913 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 447655 T1928 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 447705 T1933 C:collection1 S:shard3 oasc.ZkController.register We are http://127.0.0.1:14250/collection1/ and leader is http://127.0.0.1:14250/collection1/
   [junit4]   2> 447706 T1933 C:collection1 S:shard3 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:14250
   [junit4]   2> 447706 T1933 C:collection1 S:shard3 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 447706 T1933 C:collection1 S:shard3 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 447706 T1933 C:collection1 S:shard3 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 447707 T1901 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 447708 T1902 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14250",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:14250_",
   [junit4]   2> 	  "shard":"shard3"} current state version: 7
   [junit4]   2> 447709 T1902 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14250",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:14250_",
   [junit4]   2> 	  "shard":"shard3"}
   [junit4]   2> 447811 T1928 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 447811 T1901 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 447811 T1913 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 448664 T1875 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/cores/collection1
   [junit4]   2> 448666 T1875 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001
   [junit4]   2> 448668 T1875 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 448670 T1875 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:14256
   [junit4]   2> 448670 T1875 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, hostPort=14255, solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/tempDir-001/jetty2}
   [junit4]   2> 448671 T1875 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 448671 T1875 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/'
   [junit4]   2> 448720 T1875 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/solr.xml
   [junit4]   2> 448781 T1875 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/cores
   [junit4]   2> 448781 T1875 oasc.CoreContainer.<init> New CoreContainer 2041762435
   [junit4]   2> 448782 T1875 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/]
   [junit4]   2> 448782 T1875 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/lib
   [junit4]   2> 448783 T1875 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/lib).
   [junit4]   2> 448797 T1875 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 448797 T1875 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 448798 T1875 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 448798 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 448798 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 448799 T1875 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 448799 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 448799 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 448800 T1875 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 448800 T1875 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 448800 T1875 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 448807 T1875 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 448808 T1875 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 448809 T1875 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 448809 T1875 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 448809 T1875 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:14242/solr
   [junit4]   2> 448810 T1875 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 448810 T1875 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 448811 T1875 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 448813 T1946 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@657aa9f6 name:ZooKeeperConnection Watcher:127.0.0.1:14242 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 448814 T1875 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 448814 T1875 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 448816 T1875 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 448817 T1949 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@14f9a04e name:ZooKeeperConnection Watcher:127.0.0.1:14242/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 448818 T1875 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 448822 T1875 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 449825 T1875 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:14255_
   [junit4]   2> 449826 T1875 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:14255_
   [junit4]   2> 449829 T1875 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 449831 T1875 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/cores
   [junit4]   2> 449832 T1875 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {coreNodeName=, shard=, schema=schema.xml, dataDir=data/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/cores/collection1, name=collection1, collection=collection1, config=solrconfig.xml, loadOnStartup=true, transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/cores/collection1/}
   [junit4]   2> 449833 T1875 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/cores/collection1/
   [junit4]   2> 449833 T1875 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 449834 T1950 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 449834 T1950 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 449834 T1950 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 449834 T1901 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 449835 T1902 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14255",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:14255_",
   [junit4]   2> 	  "shard":null} current state version: 8
   [junit4]   2> 449836 T1902 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14255",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:14255_",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 449836 T1902 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 449836 T1902 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 449837 T1949 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 449837 T1913 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 449837 T1901 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 449837 T1928 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 450835 T1950 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 450836 T1950 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 450836 T1950 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 450837 T1950 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 450837 T1950 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 450837 T1950 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/cores/collection1/'
   [junit4]   2> 450855 T1950 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 450857 T1950 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 450868 T1950 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 450891 T1950 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 450941 T1950 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 451042 T1950 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 451053 T1950 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 451470 T1950 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 451486 T1950 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 451489 T1950 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 451499 T1950 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 451504 T1950 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 451508 T1950 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 451509 T1950 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 451510 T1950 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 451510 T1950 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 451511 T1950 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 451511 T1950 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 451511 T1950 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 451512 T1950 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 451512 T1950 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/worksp

[...truncated too long message...]

ent.ThreadPoolExecutor@bc189bc[Shutting down, pool size = 5, active threads = 5, queued tasks = 0, completed tasks = 48]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([457F35F1A3900CAB]:0)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:278)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #4: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=2070, name=updateExecutor-278-thread-5, state=RUNNABLE, group=TGRP-ReplicationFactorTest]
   [junit4]    > Caused by: java.util.concurrent.RejectedExecutionException: Task org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner@350152a3 rejected from java.util.concurrent.ThreadPoolExecutor@bc189bc[Shutting down, pool size = 4, active threads = 4, queued tasks = 0, completed tasks = 49]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([457F35F1A3900CAB]:0)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:278)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #5: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=2073, name=updateExecutor-278-thread-6, state=RUNNABLE, group=TGRP-ReplicationFactorTest]
   [junit4]    > Caused by: java.util.concurrent.RejectedExecutionException: Task org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner@162ec62c rejected from java.util.concurrent.ThreadPoolExecutor@bc189bc[Shutting down, pool size = 3, active threads = 3, queued tasks = 0, completed tasks = 50]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([457F35F1A3900CAB]:0)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:278)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #6: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=2076, name=updateExecutor-278-thread-7, state=RUNNABLE, group=TGRP-ReplicationFactorTest]
   [junit4]    > Caused by: java.util.concurrent.RejectedExecutionException: Task org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner@743f2543 rejected from java.util.concurrent.ThreadPoolExecutor@bc189bc[Shutting down, pool size = 2, active threads = 2, queued tasks = 0, completed tasks = 51]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([457F35F1A3900CAB]:0)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:278)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #7: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=2079, name=updateExecutor-278-thread-8, state=RUNNABLE, group=TGRP-ReplicationFactorTest]
   [junit4]    > Caused by: java.util.concurrent.RejectedExecutionException: Task org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner@43cb4a37 rejected from java.util.concurrent.ThreadPoolExecutor@bc189bc[Shutting down, pool size = 1, active threads = 1, queued tasks = 0, completed tasks = 52]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([457F35F1A3900CAB]:0)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:278)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1012391 T1875 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001
   [junit4]   2> 572143 T1874 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {multiDefault=BlockTreeOrds(blocksize=128), a_t=PostingsFormat(name=Memory doPackFST= false), text=FST50, range_facet_l=PostingsFormat(name=Memory doPackFST= false), intDefault=PostingsFormat(name=Memory doPackFST= false), timestamp=PostingsFormat(name=Memory doPackFST= false), _version_=PostingsFormat(name=Memory doPackFST= false), id=PostingsFormat(name=LuceneVarGapFixedInterval)}, docValues:{timestamp=DocValuesFormat(name=SimpleText)}, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=en_IE, timezone=America/Kentucky/Monticello
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=171993136,total=303038464
   [junit4]   2> NOTE: All tests run in this JVM: [TestCoreContainer, ChaosMonkeySafeLeaderTest, TestLMDirichletSimilarityFactory, TestSimpleTrackingShardHandler, AnalysisErrorHandlingTest, OpenExchangeRatesOrgProviderTest, TestSystemIdResolver, LeaderElectionTest, TestExactStatsCache, HighlighterMaxOffsetTest, SolrRequestParserTest, HdfsBasicDistributedZk2Test, CurrencyFieldXmlFileTest, TestMiniSolrCloudCluster, RAMDirectoryFactoryTest, TestCollectionAPI, PrimUtilsTest, HdfsCollectionsAPIDistributedZkTest, TestPostingsSolrHighlighter, TestSerializedLuceneMatchVersion, TestFieldTypeCollectionResource, TestJmxIntegration, TestPseudoReturnFields, TestSolrCoreProperties, TestImplicitCoreProperties, XsltUpdateRequestHandlerTest, CSVRequestHandlerTest, TestSolrJ, DateMathParserTest, TestTrie, TestRemoteStreaming, DistanceUnitsTest, TestEmbeddedSolrServerConstructors, CustomCollectionTest, ReplicationFactorTest]
   [junit4] Completed on J3 in 572.42s, 1 test, 1 error <<< FAILURES!

[...truncated 868 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 480 suites, 1905 tests, 1 error, 61 ignored (21 assumptions)

Total time: 60 minutes 12 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 204630843 bytes
Compression is 0.0%
Took 36 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2906 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2906/

2 tests failed.
REGRESSION:  org.apache.solr.handler.component.DistributedMLTComponentTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:35385//collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:35385//collection1
	at __randomizedtesting.SeedInfo.seed([2902CDAD2A4D727B:A156F27784B11F83]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:566)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:558)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:606)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:588)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:567)
	at org.apache.solr.handler.component.DistributedMLTComponentTest.test(DistributedMLTComponentTest.java:126)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 51 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
java.net.SocketTimeoutException: Read timed out

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: java.net.SocketTimeoutException: Read timed out
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.sendDocsWithRetry(AbstractFullDistribZkTestBase.java:774)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:277)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9961 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/init-core-data-001
   [junit4]   2> 677096 T2342 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 677103 T2342 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 677103 T2343 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 677203 T2342 oasc.ZkTestServer.run start zk server on port:18161
   [junit4]   2> 677204 T2342 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 677205 T2342 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 677210 T2350 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4d07fe53 name:ZooKeeperConnection Watcher:127.0.0.1:18161 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 677210 T2342 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 677211 T2342 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 677211 T2342 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 677215 T2342 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 677215 T2342 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 677217 T2353 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5a5efbc7 name:ZooKeeperConnection Watcher:127.0.0.1:18161/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 677217 T2342 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 677218 T2342 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 677218 T2342 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 677221 T2342 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 677222 T2342 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 677224 T2342 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 677226 T2342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 677227 T2342 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 677229 T2342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 677230 T2342 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 677332 T2342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 677332 T2342 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 677334 T2342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 677335 T2342 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 677336 T2342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 677337 T2342 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 677338 T2342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 677339 T2342 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 677340 T2342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 677341 T2342 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 677342 T2342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 677343 T2342 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 677344 T2342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 677345 T2342 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 677346 T2342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 677347 T2342 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 677348 T2342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 677349 T2342 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 678800 T2342 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores/collection1
   [junit4]   2> 678803 T2342 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 678805 T2342 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:56011
   [junit4]   2> 678805 T2342 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=23759, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores}
   [junit4]   2> 678806 T2342 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 678806 T2342 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/'
   [junit4]   2> 678841 T2342 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/solr.xml
   [junit4]   2> 678883 T2342 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores
   [junit4]   2> 678884 T2342 oasc.CoreContainer.<init> New CoreContainer 2012355157
   [junit4]   2> 678884 T2342 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/]
   [junit4]   2> 678884 T2342 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/lib
   [junit4]   2> 678885 T2342 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/lib).
   [junit4]   2> 678895 T2342 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 678895 T2342 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 678895 T2342 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 678896 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 678896 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 678896 T2342 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 678896 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 678896 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 678897 T2342 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 678897 T2342 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 678897 T2342 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 678902 T2342 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 678903 T2342 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 678903 T2342 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 678903 T2342 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 678904 T2342 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:18161/solr
   [junit4]   2> 678904 T2342 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 678904 T2342 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 678905 T2342 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 678906 T2365 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7c251cb1 name:ZooKeeperConnection Watcher:127.0.0.1:18161 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 678907 T2342 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 678907 T2342 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 678909 T2342 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 678910 T2368 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@69f080ec name:ZooKeeperConnection Watcher:127.0.0.1:18161/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 678911 T2342 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 678912 T2342 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 678913 T2342 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 678915 T2342 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 678916 T2342 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 678917 T2342 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 678919 T2342 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 678920 T2342 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 678921 T2342 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 678922 T2342 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:23759_
   [junit4]   2> 678922 T2342 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:23759_
   [junit4]   2> 678923 T2342 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 678924 T2342 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 678925 T2342 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 678926 T2342 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:23759_
   [junit4]   2> 678926 T2342 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 678927 T2342 oasc.Overseer.start Overseer (id=93613545977020419-127.0.0.1:23759_-n_0000000000) starting
   [junit4]   2> 678929 T2342 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 678933 T2342 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 678933 T2342 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 678934 T2369 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 678933 T2370 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 678936 T2342 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores
   [junit4]   2> 678938 T2342 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, schema=schema.xml, collection=control_collection, transient=false, dataDir=data/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores/collection1, shard=, coreNodeName=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores/collection1/, name=collection1, loadOnStartup=true}
   [junit4]   2> 678938 T2342 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores/collection1/
   [junit4]   2> 678939 T2342 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 678939 T2372 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 678939 T2372 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 678940 T2372 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 678940 T2368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 678941 T2369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:23759_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23759",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 0
   [junit4]   2> 678941 T2369 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:23759_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23759",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 678941 T2369 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 678942 T2369 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 678942 T2368 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 679940 T2372 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 679941 T2372 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 679941 T2372 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 679942 T2372 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 679942 T2372 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 679942 T2372 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores/collection1/'
   [junit4]   2> 679973 T2372 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 679976 T2372 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 679988 T2372 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 680014 T2372 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 680070 T2372 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 680172 T2372 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 680183 T2372 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 680625 T2372 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 680642 T2372 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 680646 T2372 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 680655 T2372 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 680660 T2372 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 680664 T2372 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 680665 T2372 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 680666 T2372 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 680666 T2372 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 680667 T2372 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 680667 T2372 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 680667 T2372 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 680667 T2372 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 680668 T2372 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores/collection1/data/
   [junit4]   2> 680668 T2372 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@542890fb
   [junit4]   2> 680669 T2372 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores/collection1/data
   [junit4]   2> 680669 T2372 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores/collection1/data/index/
   [junit4]   2> 680669 T2372 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 680670 T2372 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores/collection1/data/index
   [junit4]   2> 680670 T2372 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=13, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 680671 T2372 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@a0b4a2f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6b3358cf),segFN=segments_1,generation=1}
   [junit4]   2> 680671 T2372 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 680678 T2372 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 680678 T2372 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 680679 T2372 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 680679 T2372 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 680679 T2372 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 680679 T2372 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 680680 T2372 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 680680 T2372 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 680680 T2372 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 680700 T2372 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 680703 T2372 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 680705 T2372 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 680708 T2372 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 680713 T2372 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /replication,/admin/system,/update/json,/update,/update/json/docs,/admin/segments,/admin/luke,/admin/file,/schema,/get,/admin/mbeans,standard,/admin/threads,/admin/plugins,/admin/properties,/update/csv,/admin/ping,/config,/admin/logging
   [junit4]   2> 680715 T2372 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 680717 T2372 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 680717 T2372 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 680718 T2372 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 680719 T2372 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 680720 T2372 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1609071471, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 680721 T2372 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@a0b4a2f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6b3358cf),segFN=segments_1,generation=1}
   [junit4]   2> 680721 T2372 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 680721 T2372 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4ab3b662[collection1] main
   [junit4]   2> 680722 T2372 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 680722 T2372 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 680723 T2372 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 680723 T2372 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 680723 T2372 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 680723 T2372 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 680724 T2372 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 680724 T2372 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 680724 T2372 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 680725 T2372 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 680725 T2373 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4ab3b662[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 680726 T2372 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 680727 T2376 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:23759 collection:control_collection shard:shard1
   [junit4]   2> 680727 T2342 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 680727 T2342 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 680727 T2376 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 680729 T2342 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 680730 T2342 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 680731 T2380 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@471e5cb name:ZooKeeperConnection Watcher:127.0.0.1:18161/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 680731 T2342 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 680732 T2376 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 680732 T2342 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 680732 T2342 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 680733 T2368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 680733 T2376 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 680733 T2376 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C556 name=collection1 org.apache.solr.core.SolrCore@e402c82 url=http://127.0.0.1:23759/collection1 node=127.0.0.1:23759_ C556_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:23759_, state=down, base_url=http://127.0.0.1:23759, core=collection1}
   [junit4]   2> 680734 T2376 C:control_collection S:shard1 C556 P23759 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:23759/collection1/
   [junit4]   2> 680734 T2369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 680734 T2376 C:control_collection S:shard1 C556 P23759 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 680734 T2376 C:control_collection S:shard1 C556 P23759 oasc.SyncStrategy.syncToMe http://127.0.0.1:23759/collection1/ has no replicas
   [junit4]   2> 680734 T2376 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:23759/collection1/ shard1
   [junit4]   2> 680734 T2342 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 680735 T2376 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 680735 T2342 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 680735 T2380 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 680735 T2342 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 680735 T2368 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 680736 T2342 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 680738 T2383 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@57707db6 name:ZooKeeperConnection Watcher:127.0.0.1:18161/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 680738 T2342 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 680738 T2342 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 680740 T2368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 680741 T2369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 2, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23759",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 680742 T2369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 680743 T2369 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 680743 T2380 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 680743 T2368 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 680746 T2368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 680746 T2369 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 680747 T2368 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 680747 T2380 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 680790 T2376 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:23759/collection1/ and leader is http://127.0.0.1:23759/collection1/
   [junit4]   2> 680791 T2376 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:23759
   [junit4]   2> 680791 T2376 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 680791 T2376 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 680791 T2376 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 680793 T2368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 680794 T2369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:23759_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23759",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 4
   [junit4]   2> 680794 T2369 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:23759_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23759",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 680898 T2368 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 680898 T2380 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 681807 T2342 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores/collection1
   [junit4]   2> 681810 T2342 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001
   [junit4]   2> 681810 T2342 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 681813 T2342 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:63035
   [junit4]   2> 681813 T2342 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=15820, solrconfig=solrconfig.xml, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores}
   [junit4]   2> 681814 T2342 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 681814 T2342 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/'
   [junit4]   2> 681847 T2342 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/solr.xml
   [junit4]   2> 681886 T2342 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores
   [junit4]   2> 681887 T2342 oasc.CoreContainer.<init> New CoreContainer 915242244
   [junit4]   2> 681887 T2342 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/]
   [junit4]   2> 681887 T2342 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/lib
   [junit4]   2> 681888 T2342 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/lib).
   [junit4]   2> 681897 T2342 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 681897 T2342 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 681897 T2342 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 681898 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 681898 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 681898 T2342 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 681898 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 681898 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 681899 T2342 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 681899 T2342 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 681899 T2342 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 681904 T2342 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 681905 T2342 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 681905 T2342 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 681905 T2342 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 681905 T2342 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:18161/solr
   [junit4]   2> 681906 T2342 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 681906 T2342 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 681907 T2342 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 681908 T2395 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@510500e8 name:ZooKeeperConnection Watcher:127.0.0.1:18161 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 681909 T2342 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 681910 T2342 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 681912 T2342 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 681913 T2398 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2f1aea5 name:ZooKeeperConnection Watcher:127.0.0.1:18161/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 681914 T2342 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 681918 T2342 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 682922 T2342 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:15820_
   [junit4]   2> 682923 T2342 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:15820_
   [junit4]   2> 682926 T2342 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 682929 T2342 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores
   [junit4]   2> 682931 T2342 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {coreNodeName=, loadOnStartup=true, dataDir=data/, name=collection1, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores/collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores/collection1/, shard=, schema=schema.xml, collection=collection1, transient=false}
   [junit4]   2> 682932 T2342 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores/collection1/
   [junit4]   2> 682932 T2342 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 682934 T2399 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 682934 T2399 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 682935 T2368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 682936 T2399 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 682936 T2399 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 682936 T2369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:15820_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:15820",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 682936 T2399 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 682937 T2369 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:15820_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:15820",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1"}
   [junit4]   2> 682937 T2369 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 682937 T2369 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 682937 T2399 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 682937 T2399 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 682938 T2369 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 682939 T2398 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 682939 T2398 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 683938 T2399 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 683939 T2399 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 683939 T2399 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 683939 T2399 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 683940 T2399 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 683940 T2399 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores/collection1/'
   [junit4]   2> 683953 T2399 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 683955 T2399 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 683962 T2399 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 683977 T2399 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 684010 T2399 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 684111 T2399 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 684122 T2399 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 684548 T2399 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 684564 T2399 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 684567 T2399 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 684577 T2399 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 684582 T2399 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 684586 T2399 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 684587 T2399 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 684587 T2399 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 684587 T2399 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 684588 T2399 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 684588 T2399 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 684588 T2399 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 684588 T2399 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 684589 T2399 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 684589 T2399 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@542890fb
   [junit4]   2> 684590 T2399 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores/collection1/data
   [junit4]   2> 684590 T2399 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 684591 T2399 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 684591 T2399 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 684591 T2399 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=13, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 684592 T2399 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2758c73a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@357fc504),segFN=segments_1,generation=1}
   [junit4]   2> 684593 T2399 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 684600 T2399 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 684600 T2399 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 684600 T2399 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 684601 T2399 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 684601 T2399 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 684601 T2399 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 684602 T2399 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 684602 T2399 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 684602 T2399 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 684621 T2399 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 684624 T2399 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 684626 T2399 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 684628 T2399 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 684635 T2399 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /replication,/admin/system,/update/json,/update,/update/json/docs,/admin/segments,/admin/luke,/admin/file,/schema,/get,/admin/mbeans,standard,/admin/threads,/admin/plugins,/admin/properties,/update/csv,/admin/ping,/config,/admin/logging
   [junit4]   2> 684637 T2399 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 684639 T2399 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 684639 T2399 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 684641 T2399 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 684641 T2399 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 684642 T2399 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1609071471, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 684643 T2399 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2758c73a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@357fc504),segFN=segments_1,generation=1}
   [junit4]   2> 684643 T2399 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 684643 T2399 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@ebe3283[collection1] main
   [junit4]   2> 684644 T2399 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 684644 T2399 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 684645 T2399 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 684645 T2399 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 684645 T2399 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 684645 T2399 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 684646 T2399 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 684646 T2399 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 684646 T2399 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 684647 T2399 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 684648 T2400 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@ebe3283[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 684648 T2399 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 684649 T2403 C:collection1 S:shard2 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:15820 collection:collection1 shard:shard2
   [junit4]   2> 684649 T2342 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 684650 T2342 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 684650 T2403 C:collection1 S:shard2 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 684654 T2403 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 684656 T2368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 684656 T2403 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 684656 T2403 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C557 name=collection1 org.apache.solr.core.SolrCore@6815492e url=http://127.0.0.1:15820/collection1 node=127.0.0.1:15820_ C557_STATE=coll:collection1 core:collection1 props:{node_name=127.0.0.1:15820_, state=down, base_url=http://127.0.0.1:15820, core=collection1}
   [junit4]   2> 684656 T2403 C:collection1 S:shard2 C557 P15820 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:15820/collection1/
   [junit4]   2> 684656 T2369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 684656 T2403 C:collection1 S:shard2 C557 P15820 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 684657 T2403 C:collection1 S:shard2 C557 P15820 oasc.SyncStrategy.syncToMe http://127.0.0.1:15820/collection1/ has no replicas
   [junit4]   2> 684657 T2403 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:15820/collection1/ shard2
   [junit4]   2> 684657 T2403 C:collection1 S:shard2 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 684657 T2369 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 684658 T2398 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 684659 T2398 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 684661 T2368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 684661 T2369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:15820",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 684662 T2369 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 684663 T2398 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 684663 T2398 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 684712 T2403 C:collection1 S:shard2 oasc.ZkController.register We are http://127.0.0.1:15820/collection1/ and leader is http://127.0.0.1:15820/collection1/
   [junit4]   2> 684713 T2403 C:collection1 S:shard2 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:15820
   [junit4]   2> 684713 T2403 C:collection1 S:shard2 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 684714 T2403 C:collection1 S:shard2 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 684714 T2403 C:collection1 S:shard2 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 684715 T2368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 684716 T2369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:15820_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:15820",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 684717 T2369 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:15820_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:15820",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1"}
   [junit4]   2> 684719 T2369 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 3
   [junit4]   2> 684719 T2403 C:collection1 S:shard2 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 684720 T2398 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 684720 T2398 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 686203 T2342 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-2-001/cores/collection1
   [junit4]   2> 686206 T2342 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-2-001
   [junit4]   2> 686206 T2342 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 686209 T2342 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:26237
   [junit4]   2> 686209 T2342 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=26236, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/tempDir-001/jetty2, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-2-001/cores, solrconfig=solrconfig.xml}
   [junit4]   2> 686210 T2342 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 686210 T2342 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-2-001/'
   [junit4]   2> 686243 T2342 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-2-001/solr.xml
   [junit4]   2> 686284 T2342 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-2-001/cores
   [junit4]   2> 686285 T2342 oasc.CoreContainer.<init> New CoreContainer 1800781633
   [junit4]   2> 686285 T2342 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-2-001/]
   [junit4]   2> 686285 T2342 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-2-001/lib
   [junit4]   2> 686286 T2342 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-2-001/lib).
   [junit4]   2> 686295 T2342 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 686296 T2342 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 686296 T2342 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 686296 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 686296 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 686297 T2342 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 686297 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 686297 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 686297 T2342 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 686298 T2342 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 686298 T2342 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 686303 T2342 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 686304 T2342 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 686304 T2342 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFa

[...truncated too long message...]

lr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 2902CDAD2A4D727B-001/tempDir-001/shard2/cores/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 2902CDAD2A4D727B-001/tempDir-001/shard2/cores/collection1/data;done=false>>]
   [junit4]   2> 2026438 T7432 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 2902CDAD2A4D727B-001/tempDir-001/shard2/cores/collection1/data
   [junit4]   2> 2026439 T7432 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=DistributedMLTComponentTest -Dtests.method=test -Dtests.seed=2902CDAD2A4D727B -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ru -Dtests.timezone=Asia/Chungking -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR    104s J0 | DistributedMLTComponentTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:35385//collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([2902CDAD2A4D727B:A156F27784B11F83]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:566)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:558)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:606)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:588)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:567)
   [junit4]    > 	at org.apache.solr.handler.component.DistributedMLTComponentTest.test(DistributedMLTComponentTest.java:126)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: java.net.SocketTimeoutException: Read timed out
   [junit4]    > 	at java.net.SocketInputStream.socketRead0(Native Method)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:152)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:122)
   [junit4]    > 	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
   [junit4]    > 	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
   [junit4]    > 	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 51 more
   [junit4]   2> 2026642 T7432 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 2902CDAD2A4D727B-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {lowerfilt=PostingsFormat(name=MockRandom), intDefault=Lucene50(blocksize=128), lowerfilt1=PostingsFormat(name=Memory doPackFST= false), multiDefault=PostingsFormat(name=MockRandom), range_facet_l=PostingsFormat(name=MockRandom), timestamp=PostingsFormat(name=MockRandom), id=FSTOrd50, _version_=Lucene50(blocksize=128), lowerfilt1and2=PostingsFormat(name=MockRandom)}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=ru, timezone=Asia/Chungking
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=229808104,total=371720192
   [junit4]   2> NOTE: All tests run in this JVM: [TestRecoveryHdfs, SpellCheckComponentTest, SynonymTokenizerTest, OverseerRolesTest, SpatialHeatmapFacetsTest, TestChildDocTransformer, TestSchemaVersionResource, XmlUpdateRequestHandlerTest, BJQParserTest, TestBM25SimilarityFactory, TestExtendedDismaxParser, AddSchemaFieldsUpdateProcessorFactoryTest, TestOrdValues, ClusterStateUpdateTest, TestFastOutputStream, DeleteShardTest, TestHdfsUpdateLog, UpdateRequestProcessorFactoryTest, DirectSolrSpellCheckerTest, CoreMergeIndexesAdminHandlerTest, CoreAdminRequestStatusTest, TestLMDirichletSimilarityFactory, HdfsUnloadDistributedZkTest, SystemInfoHandlerTest, TestSimpleTrackingShardHandler, CloudMLTQParserTest, DistanceUnitsTest, TestReqParamsAPI, DisMaxRequestHandlerTest, AtomicUpdatesTest, TestDocBasedVersionConstraints, QueryResultKeyTest, TestUniqueKeyFieldResource, TestAnalyzedSuggestions, SpellCheckCollatorTest, TestStressLucene, TestCodecSupport, SchemaVersionSpecificBehaviorTest, TermsComponentTest, OverriddenZkACLAndCredentialsProvidersTest, DistributedSuggestComponentTest, SolrInfoMBeanTest, TestFieldSortValues, AddBlockUpdateTest, TestReplicaProperties, TestBlobHandler, TestFuzzyAnalyzedSuggestions, OutputWriterTest, BadCopyFieldTest, TestBinaryResponseWriter, DistributedFacetPivotSmallAdvancedTest, TestDefaultSearchFieldResource, CachingDirectoryFactoryTest, FileBasedSpellCheckerTest, TestJsonFacets, DOMUtilTest, RollingRestartTest, HttpPartitionTest, ShardSplitTest, TestCloudSchemaless, FullSolrCloudDistribCmdsTest, TestSolrConfigHandlerConcurrent, TestDynamicLoading, CollectionReloadTest, TestRebalanceLeaders, CloudExitableDirectoryReaderTest, OverseerStatusTest, TestDistribDocBasedVersion, TestShortCircuitedRequests, DistributedFacetPivotLargeTest, ZkSolrClientTest, TestHashPartitioner, TestReload, ZkCLITest, DistributedMLTComponentTest]
   [junit4] Completed on J0 in 103.81s, 1 test, 1 error <<< FAILURES!

[...truncated 413 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 479 suites, 1903 tests, 2 errors, 61 ignored (21 assumptions)

Total time: 58 minutes 57 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 212829115 bytes
Compression is 0.0%
Took 39 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2905 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2905/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:21000/p_se/a/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:21000/p_se/a/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([DFC12658F5827783:579519825B7E1A7B]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:21000 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28440/_xv/b/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28440/_xv/b/collection1
	at __randomizedtesting.SeedInfo.seed([DFC12658F5827783:579519825B7E1A7B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28440/_xv/b/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:28440 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22580/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22580/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([DFC12658F5827783:579519825B7E1A7B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22580/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:22580 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9544 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/init-core-data-001
   [junit4]   2> 228752 T610 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 228762 T610 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 228763 T611 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 228862 T610 oasc.ZkTestServer.run start zk server on port:22563
   [junit4]   2> 228863 T610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 228864 T610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 228873 T618 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@37715099 name:ZooKeeperConnection Watcher:127.0.0.1:22563 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 228873 T610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 228874 T610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 228874 T610 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 228878 T610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 228879 T610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 228882 T621 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@42f758cf name:ZooKeeperConnection Watcher:127.0.0.1:22563/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 228882 T610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 228883 T610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 228883 T610 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 228886 T610 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 228888 T610 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 228890 T610 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 228893 T610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 228895 T610 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 228900 T610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 228901 T610 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 228905 T610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 228906 T610 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 228909 T610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 228911 T610 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 228915 T610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 228916 T610 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 228920 T610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 228922 T610 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 228927 T610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 228929 T610 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 228933 T610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 228935 T610 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 228939 T610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 228940 T610 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 228947 T610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 228949 T610 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 228954 T610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 228955 T610 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 233138 T610 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores/collection1
   [junit4]   2> 233142 T610 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 233145 T610 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:22581
   [junit4]   2> 233145 T610 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/tempDir-001/control/data, hostContext=/, hostPort=22580, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores}
   [junit4]   2> 233146 T610 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 233146 T610 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/'
   [junit4]   2> 233194 T610 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/solr.xml
   [junit4]   2> 233407 T610 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores
   [junit4]   2> 233408 T610 oasc.CoreContainer.<init> New CoreContainer 957384846
   [junit4]   2> 233408 T610 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/]
   [junit4]   2> 233409 T610 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/lib
   [junit4]   2> 233413 T610 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/lib).
   [junit4]   2> 233446 T610 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 233446 T610 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 233447 T610 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 233447 T610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 233447 T610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 233448 T610 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 233448 T610 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 233449 T610 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 233450 T610 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 233450 T610 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 233450 T610 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 233475 T610 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 233478 T610 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 233478 T610 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 233478 T610 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 233479 T610 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:22563/solr
   [junit4]   2> 233479 T610 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 233480 T610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 233482 T610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 233484 T633 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2258256e name:ZooKeeperConnection Watcher:127.0.0.1:22563 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 233485 T610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 233486 T610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 233494 T610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 233496 T636 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6b2e0f34 name:ZooKeeperConnection Watcher:127.0.0.1:22563/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 233496 T610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 233499 T610 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 233506 T610 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 233512 T610 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 233517 T610 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 233521 T610 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 233534 T610 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 233542 T610 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 233547 T610 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 233552 T610 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:22580_
   [junit4]   2> 233553 T610 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:22580_
   [junit4]   2> 233557 T610 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 233560 T610 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 233565 T610 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 233569 T610 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:22580_
   [junit4]   2> 233569 T610 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 233574 T610 oasc.Overseer.start Overseer (id=93612242919948291-127.0.0.1:22580_-n_0000000000) starting
   [junit4]   2> 233579 T610 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 233594 T610 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 233595 T638 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 233595 T610 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 233596 T637 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 233603 T610 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores
   [junit4]   2> 233611 T610 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {loadOnStartup=true, coreNodeName=, schema=schema.xml, transient=false, dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores/collection1/, shard=, config=solrconfig.xml, collection=control_collection, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores/collection1, name=collection1}
   [junit4]   2> 233612 T610 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores/collection1/
   [junit4]   2> 233612 T610 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 233614 T640 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 233614 T640 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 233616 T636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 233616 T640 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 233618 T637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22580",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:22580_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null} current state version: 0
   [junit4]   2> 233620 T637 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22580",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:22580_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 233620 T637 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 233620 T637 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 233624 T636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 234617 T640 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 234617 T640 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 234619 T640 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 234620 T640 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 234621 T640 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 234621 T640 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores/collection1/'
   [junit4]   2> 234651 T640 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 234658 T640 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 234686 T640 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 234753 T640 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 234888 T640 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 234890 T640 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 234913 T640 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 235913 T640 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 235952 T640 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 235957 T640 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 235977 T640 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 235986 T640 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 235993 T640 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 235995 T640 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 235996 T640 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 235997 T640 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 235998 T640 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 235998 T640 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 235999 T640 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 235999 T640 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 236000 T640 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores/collection1/data/
   [junit4]   2> 236000 T640 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7f14be2e
   [junit4]   2> 236001 T640 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores/collection1/data
   [junit4]   2> 236002 T640 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores/collection1/data/index/
   [junit4]   2> 236002 T640 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 236003 T640 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores/collection1/data/index
   [junit4]   2> 236004 T640 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=34, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6812259126929583]
   [junit4]   2> 236005 T640 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7f765e23 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@341e8d35),segFN=segments_1,generation=1}
   [junit4]   2> 236005 T640 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 236014 T640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 236014 T640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 236015 T640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 236015 T640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 236015 T640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 236016 T640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 236017 T640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 236017 T640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 236017 T640 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 236039 T640 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 236041 T640 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 236043 T640 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 236045 T640 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 236049 T640 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /schema,/admin/system,/admin/file,/admin/logging,/admin/luke,standard,/update/csv,/admin/ping,/update/json,/admin/threads,/admin/mbeans,/admin/properties,/config,/get,/admin/segments,/update,/replication,/update/json/docs,/admin/plugins
   [junit4]   2> 236051 T640 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 236053 T640 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 236053 T640 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 236055 T640 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 236055 T640 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 236055 T640 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=46, maxMergeAtOnceExplicit=32, maxMergedSegmentMB=50.4541015625, floorSegmentMB=0.912109375, forceMergeDeletesPctAllowed=9.525263553538434, segmentsPerTier=24.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.16973219653750882
   [junit4]   2> 236056 T640 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7f765e23 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@341e8d35),segFN=segments_1,generation=1}
   [junit4]   2> 236056 T640 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 236057 T640 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@6b9df4b3[collection1] main
   [junit4]   2> 236057 T640 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 236058 T640 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 236058 T640 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 236059 T640 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 236059 T640 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 236059 T640 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 236060 T640 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 236060 T640 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 236060 T640 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 236061 T640 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 236061 T641 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6b9df4b3[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 236062 T640 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 236063 T644 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:22580 collection:control_collection shard:shard1
   [junit4]   2> 236063 T610 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 236063 T610 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 236063 T644 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 236067 T644 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 236068 T610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 236068 T636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 236068 T644 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 236068 T610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 236069 T644 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C67 name=collection1 org.apache.solr.core.SolrCore@347de428 url=http://127.0.0.1:22580/collection1 node=127.0.0.1:22580_ C67_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:22580, core=collection1, node_name=127.0.0.1:22580_}
   [junit4]   2> 236069 T644 C:control_collection S:shard1 C67 P22580 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:22580/collection1/
   [junit4]   2> 236070 T644 C:control_collection S:shard1 C67 P22580 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 236070 T644 C:control_collection S:shard1 C67 P22580 oasc.SyncStrategy.syncToMe http://127.0.0.1:22580/collection1/ has no replicas
   [junit4]   2> 236070 T637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 236070 T648 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4b4907d8 name:ZooKeeperConnection Watcher:127.0.0.1:22563/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 236070 T644 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:22580/collection1/ shard1
   [junit4]   2> 236070 T610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 236071 T644 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 236071 T636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 236071 T610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 236072 T610 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 236074 T636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 236074 T610 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 236075 T637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22580",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 236178 T636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 236178 T648 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 236227 T644 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:22580/collection1/ and leader is http://127.0.0.1:22580/collection1/
   [junit4]   2> 236228 T644 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:22580
   [junit4]   2> 236228 T644 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 236228 T644 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 236228 T644 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 236230 T636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 236231 T637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22580",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:22580_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null} current state version: 3
   [junit4]   2> 236231 T637 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22580",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:22580_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 236334 T636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 236334 T648 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 239237 T610 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores/collection1
   [junit4]   2> 239249 T610 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001
   [junit4]   2> 239251 T610 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 239259 T610 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:22602
   [junit4]   2> 239259 T610 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/tempDir-001/jetty1, hostContext=/, solrconfig=solrconfig.xml, hostPort=22601}
   [junit4]   2> 239260 T610 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 239261 T610 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/'
   [junit4]   2> 239405 T610 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/solr.xml
   [junit4]   2> 239472 T610 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores
   [junit4]   2> 239472 T610 oasc.CoreContainer.<init> New CoreContainer 1684710647
   [junit4]   2> 239473 T610 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/]
   [junit4]   2> 239473 T610 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/lib
   [junit4]   2> 239474 T610 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/lib).
   [junit4]   2> 239487 T610 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 239488 T610 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 239488 T610 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 239489 T610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 239489 T610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 239489 T610 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 239489 T610 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 239490 T610 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 239490 T610 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 239490 T610 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 239491 T610 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 239497 T610 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 239498 T610 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 239498 T610 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 239499 T610 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 239499 T610 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:22563/solr
   [junit4]   2> 239500 T610 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 239500 T610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 239501 T610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 239504 T660 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@74ed31e5 name:ZooKeeperConnection Watcher:127.0.0.1:22563 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 239504 T610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 239505 T610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 239509 T610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 239512 T663 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4faae9f3 name:ZooKeeperConnection Watcher:127.0.0.1:22563/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 239512 T610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 239518 T610 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 240529 T610 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:22601_
   [junit4]   2> 240531 T610 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:22601_
   [junit4]   2> 240543 T610 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 240551 T610 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores
   [junit4]   2> 240560 T610 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores/collection1/, collection=collection1, name=collection1, shard=, loadOnStartup=true, schema=schema.xml, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores/collection1, config=solrconfig.xml, dataDir=data/}
   [junit4]   2> 240561 T610 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores/collection1/
   [junit4]   2> 240562 T610 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 240563 T664 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 240564 T664 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 240566 T664 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 240566 T636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 240567 T637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22601",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:22601_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null} current state version: 4
   [junit4]   2> 240568 T637 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22601",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:22601_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 240568 T637 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 240569 T637 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 240572 T636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 240572 T648 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 240573 T663 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 241566 T664 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 241567 T664 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 241568 T664 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 241569 T664 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 241569 T664 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 241570 T664 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores/collection1/'
   [junit4]   2> 241604 T664 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 241615 T664 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 241641 T664 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 241698 T664 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 241785 T664 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 241787 T664 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 241807 T664 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 242758 T664 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 242775 T664 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 242778 T664 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 242812 T664 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 242817 T664 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 242821 T664 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 242822 T664 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 242822 T664 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 242823 T664 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 242823 T664 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 242823 T664 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 242824 T664 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 242824 T664 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 242824 T664 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 242825 T664 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7f14be2e
   [junit4]   2> 242825 T664 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores/collection1/data
   [junit4]   2> 242826 T664 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 242826 T664 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 242826 T664 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 242827 T664 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=34, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6812259126929583]
   [junit4]   2> 242828 T664 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3d1ff8dc lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7e0b52dd),segFN=segments_1,generation=1}
   [junit4]   2> 242828 T664 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 242834 T664 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 242834 T664 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 242835 T664 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 242835 T664 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 242835 T664 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 242835 T664 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 242836 T664 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 242836 T664 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 242836 T664 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 242853 T664 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 242855 T664 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 242857 T664 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 242859 T664 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 242867 T664 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /schema,/admin/system,/admin/file,/admin/logging,/admin/luke,standard,/update/csv,/admin/ping,/update/json,/admin/threads,/admin/mbeans,/admin/properties,/config,/get,/admin/segments,/update,/replication,/update/json/docs,/admin/plugins
   [junit4]   2> 242869 T664 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 242871 T664 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 242871 T664 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 242873 T664 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 242873 T664 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 242874 T664 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=46, maxMergeAtOnceExplicit=32, maxMergedSegmentMB=50.4541015625, floorSegmentMB=0.912109375, forceMergeDeletesPctAllowed=9.525263553538434, segmentsPerTier=24.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.16973219653750882
   [junit4]   2> 242876 T664 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3d1ff8dc lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7e0b52dd),segFN=segments_1,generation=1}
   [junit4]   2> 242876 T664 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 242876 T664 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@63471445[collection1] main
   [junit4]   2> 242877 T664 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 242880 T664 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 242880 T664 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 242884 T664 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 242884 T664 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 242884 T664 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 242885 T664 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 242885 T664 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 242886 T664 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 242886 T664 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 242888 T665 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@63471445[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 242889 T664 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 242890 T668 C:collection1 S:shard3 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:22601 collection:collection1 shard:shard3
   [junit4]   2> 242891 T610 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 242892 T610 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 242892 T668 C:collection1 S:shard3 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
   [junit4]   2> 242905 T668 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard3
   [junit4]   2> 242908 T636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 242909 T668 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 242909 T668 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C68 name=collection1 org.apache.solr.core.SolrCore@65a55801 url=http://127.0.0.1:22601/collection1 node=127.0.0.1:22601_ C68_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:22601, core=collection1, node_name=127.0.0.1:22601_}
   [junit4]   2> 242910 T668 C:collection1 S:shard3 C68 P22601 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:22601/collection1/
   [junit4]   2> 242910 T668 C:collection1 S:shard3 C68 P22601 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 242910 T637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 242911 T668 C:collection1 S:shard3 C68 P22601 oasc.SyncStrategy.syncToMe http://127.0.0.1:22601/collection1/ has no replicas
   [junit4]   2> 242912 T668 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:22601/collection1/ shard3
   [junit4]   2> 242912 T668 C:collection1 S:shard3 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard3
   [junit4]   2> 242918 T648 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 242919 T663 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 242918 T636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 242924 T636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 242926 T637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22601",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 243035 T636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 243035 T648 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 243035 T663 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 243079 T668 C:collection1 S:shard3 oasc.ZkController.register We are http://127.0.0.1:22601/collection1/ and leader is http://127.0.0.1:22601/collection1/
   [junit4]   2> 243079 T668 C:collection1 S:shard3 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:22601
   [junit4]   2> 243080 T668 C:collection1 S:shard3 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 243080 T668 C:collection1 S:shard3 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 243080 T668 C:collection1 S:shard3 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 243083 T636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 243085 T637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22601",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:22601_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null} current state version: 7
   [junit4]   2> 243086 T637 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22601",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:22601_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 243194 T636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 243195 T648 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 243194 T663 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 247015 T610 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-2-001/cores/collection1
   [junit4]   2> 247025 T610 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/sol

[...truncated too long message...]

etContextHandler{/p_se/a,null}
   [junit4]   2> 1487445 T2508 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:20927 20927
   [junit4]   2> 1487791 T2509 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:20927 20927
   [junit4]   2> 1487793 T2509 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1487793 T2508 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:21008/p_se/a
   [junit4]   2> 1487793 T2508 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 11 connections to: http://127.0.0.1:20963/p_se/a
   [junit4]   2> 1487794 T2508 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:20980/p_se/a
   [junit4]   2> 1487794 T2508 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:21000/p_se/a
   [junit4]   2> 1487794 T2508 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:20951/p_se/a
   [junit4]   2> 1487795 T2508 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:21008/p_se/a
   [junit4]   2> 1487795 T2508 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:20963/p_se/a
   [junit4]   2> 1487795 T2508 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:20980/p_se/a
   [junit4]   2> 1487795 T2508 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:21000/p_se/a
   [junit4]   2> 1487796 T2508 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:20951/p_se/a
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=DFC12658F5827783 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=en_PH -Dtests.timezone=America/Paramaribo -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   92.8s J0 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:21000/p_se/a/c8n_1x3_commits_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([DFC12658F5827783:579519825B7E1A7B]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:21000 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 48 more
   [junit4]   2> 1487804 T2508 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DFC12658F5827783-001
   [junit4]   2> 92813 T2507 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=DefaultSimilarity, locale=en_PH, timezone=America/Paramaribo
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=85241336,total=286261248
   [junit4]   2> NOTE: All tests run in this JVM: [SolrInfoMBeanTest, JsonLoaderTest, TestJmxIntegration, ShardRoutingTest, ExitableDirectoryReaderTest, CSVRequestHandlerTest, TestSweetSpotSimilarityFactory, TestSuggestSpellingConverter, HighlighterMaxOffsetTest, OverseerRolesTest, TestLuceneMatchVersion, ReplicationFactorTest, TestShortCircuitedRequests, DateRangeFieldTest, DocumentBuilderTest, CacheHeaderTest, RecoveryZkTest, TestInitParams, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestJettySolrRunner, TestSerializedLuceneMatchVersion, TestSchemaResource, TestTrackingShardHandlerFactory, PathHierarchyTokenizerFactoryTest, TestDynamicFieldResource, BasicFunctionalityTest, RequiredFieldsTest, SimpleCollectionCreateDeleteTest, TestExactSharedStatsCache, TestFileDictionaryLookup, TestFunctionQuery, TestClusterStateMutator, TestManagedResourceStorage, AnalysisErrorHandlingTest, RemoteQueryErrorTest, TestSolrDeletionPolicy2, HttpPartitionTest, AnalyticsQueryTest, TestTrie, XsltUpdateRequestHandlerTest, TestConfigOverlay, TestInitQParser, ScriptEngineTest, TestCollapseQParserPlugin, DirectUpdateHandlerTest, TestQueryTypes, TestRecovery, SolrIndexSplitterTest, AliasIntegrationTest, TestReloadAndDeleteDocs, HdfsDirectoryTest, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J0 in 93.60s, 1 test, 1 error <<< FAILURES!

[...truncated 748 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 479 suites, 1903 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 61 minutes 4 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 213820630 bytes
Compression is 0.0%
Took 41 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2904 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2904/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:32152/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:32152/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([998831C6A7E2310A:11DC0E1C091E5CF2]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:32152 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37941/cpwc/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37941/cpwc/collection1
	at __randomizedtesting.SeedInfo.seed([998831C6A7E2310A:11DC0E1C091E5CF2]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37941/cpwc/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:37941 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:31600/_bkj/mq/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:31600/_bkj/mq/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([998831C6A7E2310A:11DC0E1C091E5CF2]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:31600/_bkj/mq/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:31600 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9355 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/init-core-data-001
   [junit4]   2> 160347 T395 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 160352 T395 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 160353 T396 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 160453 T395 oasc.ZkTestServer.run start zk server on port:32104
   [junit4]   2> 160454 T395 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 160455 T395 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 160458 T403 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6c3d4223 name:ZooKeeperConnection Watcher:127.0.0.1:32104 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 160458 T395 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 160459 T395 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 160459 T395 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 160462 T395 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 160462 T395 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 160463 T406 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6c4b078f name:ZooKeeperConnection Watcher:127.0.0.1:32104/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 160464 T395 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 160464 T395 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 160464 T395 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 160466 T395 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 160468 T395 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 160469 T395 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 160471 T395 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 160472 T395 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 160475 T395 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 160475 T395 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 160478 T395 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 160478 T395 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 160480 T395 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 160480 T395 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 160483 T395 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 160483 T395 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 160485 T395 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 160486 T395 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 160488 T395 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 160489 T395 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 160490 T395 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 160491 T395 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 160492 T395 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 160493 T395 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 160495 T395 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 160495 T395 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 160497 T395 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 160497 T395 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 161891 T395 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores/collection1
   [junit4]   2> 161896 T395 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 161900 T395 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:32111
   [junit4]   2> 161900 T395 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=32110, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/tempDir-001/control/data}
   [junit4]   2> 161901 T395 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 161901 T395 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/'
   [junit4]   2> 161953 T395 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/solr.xml
   [junit4]   2> 162016 T395 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores
   [junit4]   2> 162017 T395 oasc.CoreContainer.<init> New CoreContainer 139305003
   [junit4]   2> 162017 T395 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/]
   [junit4]   2> 162018 T395 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/lib
   [junit4]   2> 162019 T395 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/lib).
   [junit4]   2> 162033 T395 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 162033 T395 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 162033 T395 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 162034 T395 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 162034 T395 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 162034 T395 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 162035 T395 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 162035 T395 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 162035 T395 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 162035 T395 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 162036 T395 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 162043 T395 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 162045 T395 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 162045 T395 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 162045 T395 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 162046 T395 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:32104/solr
   [junit4]   2> 162046 T395 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 162046 T395 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 162047 T395 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 162049 T418 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7a84a043 name:ZooKeeperConnection Watcher:127.0.0.1:32104 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 162049 T395 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 162050 T395 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 162052 T395 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 162054 T421 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@22497601 name:ZooKeeperConnection Watcher:127.0.0.1:32104/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 162054 T395 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 162056 T395 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 162059 T395 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 162061 T395 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 162063 T395 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 162066 T395 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 162068 T395 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 162070 T395 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 162072 T395 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 162073 T395 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32110_
   [junit4]   2> 162074 T395 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:32110_
   [junit4]   2> 162076 T395 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 162077 T395 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 162079 T395 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 162081 T395 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:32110_
   [junit4]   2> 162081 T395 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 162083 T395 oasc.Overseer.start Overseer (id=93611275814240259-127.0.0.1:32110_-n_0000000000) starting
   [junit4]   2> 162085 T395 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 162092 T395 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 162093 T423 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 162093 T395 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 162093 T422 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 162098 T395 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores
   [junit4]   2> 162101 T395 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores/collection1/, shard=, name=collection1, schema=schema.xml, config=solrconfig.xml, loadOnStartup=true, transient=false, collection=control_collection, coreNodeName=, dataDir=data/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores/collection1}
   [junit4]   2> 162101 T395 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores/collection1/
   [junit4]   2> 162101 T395 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 162103 T425 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 162103 T425 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 162104 T421 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 162104 T425 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 162105 T422 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32110",
   [junit4]   2> 	  "node_name":"127.0.0.1:32110_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null} current state version: 0
   [junit4]   2> 162106 T422 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32110",
   [junit4]   2> 	  "node_name":"127.0.0.1:32110_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 162106 T422 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 162106 T422 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 162108 T421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 163105 T425 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 163106 T425 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 163107 T425 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 163107 T425 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 163108 T425 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 163108 T425 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores/collection1/'
   [junit4]   2> 163127 T425 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 163130 T425 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 163145 T425 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 163175 T425 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 163278 T425 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 163280 T425 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 163297 T425 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 163690 T425 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 163703 T425 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 163706 T425 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 163722 T425 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 163727 T425 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 163730 T425 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 163731 T425 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 163732 T425 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 163732 T425 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 163733 T425 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 163733 T425 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 163733 T425 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 163733 T425 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 163734 T425 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores/collection1/data/
   [junit4]   2> 163734 T425 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4f3fd958
   [junit4]   2> 163734 T425 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores/collection1/data
   [junit4]   2> 163735 T425 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores/collection1/data/index/
   [junit4]   2> 163735 T425 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 163735 T425 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores/collection1/data/index
   [junit4]   2> 163736 T425 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=73840210, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 163736 T425 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@270b6d1c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5f285d4a),segFN=segments_1,generation=1}
   [junit4]   2> 163736 T425 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 163741 T425 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 163741 T425 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 163741 T425 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 163742 T425 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 163742 T425 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 163742 T425 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 163742 T425 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 163742 T425 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 163743 T425 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 163757 T425 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 163759 T425 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 163760 T425 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 163762 T425 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 163766 T425 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/luke,/replication,/get,/admin/mbeans,/admin/properties,/update/csv,/update/json,/admin/logging,/admin/system,/admin/threads,/update,/admin/ping,/schema,/admin/plugins,/update/json/docs,/admin/segments,/config,standard,/admin/file
   [junit4]   2> 163768 T425 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 163769 T425 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 163769 T425 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 163770 T425 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 163771 T425 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 163771 T425 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=38, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.10602606676023907]
   [junit4]   2> 163771 T425 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@270b6d1c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5f285d4a),segFN=segments_1,generation=1}
   [junit4]   2> 163772 T425 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 163772 T425 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@7a00e70c[collection1] main
   [junit4]   2> 163772 T425 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 163773 T425 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 163773 T425 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 163774 T425 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 163774 T425 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 163774 T425 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 163775 T425 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 163775 T425 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 163775 T425 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 163775 T425 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 163776 T426 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7a00e70c[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 163777 T425 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 163778 T429 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:32110 collection:control_collection shard:shard1
   [junit4]   2> 163778 T395 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 163779 T395 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 163779 T429 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 163784 T429 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 163785 T395 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 163785 T395 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 163786 T421 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 163787 T429 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 163787 T433 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@425a03e7 name:ZooKeeperConnection Watcher:127.0.0.1:32104/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 163787 T429 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 163787 T395 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 163787 T422 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C107 name=collection1 org.apache.solr.core.SolrCore@7125ade5 url=http://127.0.0.1:32110/collection1 node=127.0.0.1:32110_ C107_STATE=coll:control_collection core:collection1 props:{core=collection1, base_url=http://127.0.0.1:32110, node_name=127.0.0.1:32110_, state=down}
   [junit4]   2> 163787 T429 C:control_collection S:shard1 C107 P32110 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:32110/collection1/
   [junit4]   2> 163788 T395 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 163788 T429 C:control_collection S:shard1 C107 P32110 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 163789 T395 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 163789 T429 C:control_collection S:shard1 C107 P32110 oasc.SyncStrategy.syncToMe http://127.0.0.1:32110/collection1/ has no replicas
   [junit4]   2> 163789 T429 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:32110/collection1/ shard1
   [junit4]   2> 163789 T421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 163790 T429 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 163792 T395 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 163795 T421 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 163796 T422 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32110",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 163903 T433 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 163903 T421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 163947 T429 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:32110/collection1/ and leader is http://127.0.0.1:32110/collection1/
   [junit4]   2> 163948 T429 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:32110
   [junit4]   2> 163948 T429 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 163948 T429 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 163948 T429 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 163950 T421 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 163951 T422 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32110",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:32110_",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1"} current state version: 3
   [junit4]   2> 163952 T422 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32110",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:32110_",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 164055 T421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 164055 T433 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 166294 T395 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores/collection1
   [junit4]   2> 166298 T395 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001
   [junit4]   2> 166299 T395 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 166302 T395 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:32132
   [junit4]   2> 166303 T395 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores, hostPort=32131, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/tempDir-001/jetty1, solrconfig=solrconfig.xml}
   [junit4]   2> 166304 T395 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 166305 T395 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/'
   [junit4]   2> 166365 T395 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/solr.xml
   [junit4]   2> 166431 T395 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores
   [junit4]   2> 166431 T395 oasc.CoreContainer.<init> New CoreContainer 1214137018
   [junit4]   2> 166431 T395 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/]
   [junit4]   2> 166432 T395 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/lib
   [junit4]   2> 166433 T395 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/lib).
   [junit4]   2> 166448 T395 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 166449 T395 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 166449 T395 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 166449 T395 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 166450 T395 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 166450 T395 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 166450 T395 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 166451 T395 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 166451 T395 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 166451 T395 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 166452 T395 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 166460 T395 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 166462 T395 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 166462 T395 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 166462 T395 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 166463 T395 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:32104/solr
   [junit4]   2> 166463 T395 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 166463 T395 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 166464 T395 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 166467 T445 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@48885506 name:ZooKeeperConnection Watcher:127.0.0.1:32104 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 166467 T395 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 166468 T395 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 166471 T395 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 166473 T448 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@b27c214 name:ZooKeeperConnection Watcher:127.0.0.1:32104/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 166474 T395 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 166482 T395 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 167488 T395 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32131_
   [junit4]   2> 167489 T395 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:32131_
   [junit4]   2> 167491 T395 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 167494 T395 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores
   [junit4]   2> 167496 T395 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, name=collection1, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores/collection1, schema=schema.xml, loadOnStartup=true, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores/collection1/, collection=collection1, dataDir=data/, coreNodeName=, shard=}
   [junit4]   2> 167497 T395 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores/collection1/
   [junit4]   2> 167497 T395 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 167498 T449 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 167498 T449 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 167499 T421 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 167499 T449 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 167500 T422 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32131",
   [junit4]   2> 	  "node_name":"127.0.0.1:32131_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null} current state version: 4
   [junit4]   2> 167500 T422 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32131",
   [junit4]   2> 	  "node_name":"127.0.0.1:32131_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 167500 T422 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 167501 T422 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 167502 T421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 167502 T433 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 167502 T448 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 168499 T449 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 168500 T449 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 168501 T449 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 168501 T449 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 168502 T449 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 168502 T449 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores/collection1/'
   [junit4]   2> 168521 T449 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 168524 T449 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 168538 T449 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 168567 T449 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 168624 T449 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 168626 T449 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 168640 T449 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 169025 T449 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 169039 T449 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 169041 T449 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 169054 T449 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 169058 T449 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 169061 T449 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 169062 T449 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 169062 T449 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 169062 T449 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 169063 T449 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 169063 T449 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 169063 T449 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 169064 T449 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 169064 T449 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 169064 T449 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4f3fd958
   [junit4]   2> 169065 T449 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores/collection1/data
   [junit4]   2> 169065 T449 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 169065 T449 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 169065 T449 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 169066 T449 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=73840210, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 169067 T449 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1abcd676 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@63a3eab0),segFN=segments_1,generation=1}
   [junit4]   2> 169067 T449 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 169073 T449 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 169073 T449 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 169073 T449 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 169074 T449 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 169074 T449 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 169074 T449 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 169074 T449 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 169075 T449 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 169075 T449 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 169094 T449 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 169096 T449 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 169097 T449 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 169099 T449 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 169105 T449 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/luke,/replication,/get,/admin/mbeans,/admin/properties,/update/csv,/update/json,/admin/logging,/admin/system,/admin/threads,/update,/admin/ping,/schema,/admin/plugins,/update/json/docs,/admin/segments,/config,standard,/admin/file
   [junit4]   2> 169107 T449 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 169108 T449 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 169108 T449 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 169110 T449 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 169110 T449 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 169111 T449 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=38, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.10602606676023907]
   [junit4]   2> 169111 T449 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1abcd676 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@63a3eab0),segFN=segments_1,generation=1}
   [junit4]   2> 169111 T449 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 169111 T449 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@6a3c49d3[collection1] main
   [junit4]   2> 169112 T449 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 169113 T449 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 169113 T449 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 169113 T449 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 169113 T449 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 169114 T449 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 169114 T449 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 169114 T449 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 169114 T449 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 169115 T449 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 169116 T450 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6a3c49d3[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 169116 T449 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 169117 T453 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:32131 collection:collection1 shard:shard1
   [junit4]   2> 169117 T395 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 169118 T395 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 169118 T453 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 169122 T453 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 169123 T421 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 169124 T453 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 169124 T453 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C108 name=collection1 org.apache.solr.core.SolrCore@210cd041 url=http://127.0.0.1:32131/collection1 node=127.0.0.1:32131_ C108_STATE=coll:collection1 core:collection1 props:{core=collection1, base_url=http://127.0.0.1:32131, node_name=127.0.0.1:32131_, state=down}
   [junit4]   2> 169124 T453 C:collection1 S:shard1 C108 P32131 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:32131/collection1/
   [junit4]   2> 169124 T422 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 169125 T453 C:collection1 S:shard1 C108 P32131 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 169125 T453 C:collection1 S:shard1 C108 P32131 oasc.SyncStrategy.syncToMe http://127.0.0.1:32131/collection1/ has no replicas
   [junit4]   2> 169125 T453 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:32131/collection1/ shard1
   [junit4]   2> 169125 T453 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 169126 T433 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 169126 T421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 169126 T448 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 169130 T421 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 169131 T422 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32131",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 169234 T421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 169234 T448 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 169234 T433 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 169283 T453 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:32131/collection1/ and leader is http://127.0.0.1:32131/collection1/
   [junit4]   2> 169284 T453 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:32131
   [junit4]   2> 169284 T453 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 169284 T453 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 169284 T453 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 169285 T421 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 169286 T422 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32131",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:32131_",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1"} current state version: 7
   [junit4]   2> 169287 T422 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32131",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:32131_",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 169390 T421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 169390 T448 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 169390 T433 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstat

[...truncated too long message...]

ue
   [junit4]   2> 		4	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1504270 T3781 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 10 connections to: http://127.0.0.1:31600/_bkj/mq
   [junit4]   2> 1504271 T3781 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:31574/_bkj/mq
   [junit4]   2> 1504271 T3781 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 1 connections to: http://127.0.0.1:31559/_bkj/mq
   [junit4]   2> 1504271 T3781 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:31585/_bkj/mq
   [junit4]   2> 1504271 T3781 C:control_collection S:shard1 c:collection1 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1504272 T3781 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:31600/_bkj/mq
   [junit4]   2> 1504272 T3781 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:31574/_bkj/mq
   [junit4]   2> 1504272 T3781 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:31559/_bkj/mq
   [junit4]   2> 1504272 T3781 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:31585/_bkj/mq
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=998831C6A7E2310A -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ro -Dtests.timezone=SystemV/EST5 -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   80.1s J1 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:31600/_bkj/mq/repfacttest_c8n_1x3_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([998831C6A7E2310A:11DC0E1C091E5CF2]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:31600/_bkj/mq/repfacttest_c8n_1x3_shard1_replica1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:31600 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 53 more
   [junit4]   2> 1504288 T3781 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 998831C6A7E2310A-001
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=ro, timezone=SystemV/EST5
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=46987136,total=306708480
   [junit4]   2> NOTE: All tests run in this JVM: [DeleteReplicaTest, TestDynamicFieldResource, SearchHandlerTest, SpellPossibilityIteratorTest, TestSolrCoreProperties, ShardRoutingTest, ExternalFileFieldSortTest, TestAnalyzeInfixSuggestions, RequiredFieldsTest, TestRTGBase, LeaderFailoverAfterPartitionTest, XmlUpdateRequestHandlerTest, LeaderElectionIntegrationTest, DeleteInactiveReplicaTest, TestEmbeddedSolrServerConstructors, AddBlockUpdateTest, TestHashQParserPlugin, TestManagedSchema, TestManagedStopFilterFactory, DocumentAnalysisRequestHandlerTest, TestCSVLoader, TestCursorMarkWithoutUniqueKey, FastVectorHighlighterTest, TestDistributedGrouping, BadIndexSchemaTest, HardAutoCommitTest, TestRawResponseWriter, DateMathParserTest, InfoHandlerTest, UnloadDistributedZkTest, TestMacros, TestLMDirichletSimilarityFactory, DocValuesMissingTest, TestSchemaSimilarityResource, TestConfigSets, TestTrackingShardHandlerFactory, SolrXmlInZkTest, TestAnalyzedSuggestions, ChaosMonkeySafeLeaderTest, TestHashPartitioner, TestSchemaVersionResource, TestReqParamsAPI, SliceStateTest, SegmentsInfoRequestHandlerTest, RAMDirectoryFactoryTest, LukeRequestHandlerTest, DistributedFacetPivotLongTailTest, HdfsCollectionsAPIDistributedZkTest, TestPartialUpdateDeduplication, DistanceUnitsTest, MBeansHandlerTest, TestFiltering, TestInfoStreamLogging, TestCollationFieldDocValues, TestLuceneMatchVersion, TestRandomFaceting, DistributedQueueTest, RecoveryZkTest, OpenCloseCoreStressTest, BasicDistributedZkTest, ReplicationFactorTest]
   [junit4] Completed on J1 in 80.20s, 1 test, 1 error <<< FAILURES!

[...truncated 712 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 479 suites, 1903 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 54 minutes 3 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 216312738 bytes
Compression is 0.0%
Took 42 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2903 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2903/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:34770/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:34770/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([9D5B81451A89ABB3:150FBE9FB475C64B]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:34770 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:34979 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:34979 failed to respond
	at __randomizedtesting.SeedInfo.seed([9D5B81451A89ABB3:150FBE9FB475C64B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:34979 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:54113 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:54113 failed to respond
	at __randomizedtesting.SeedInfo.seed([9D5B81451A89ABB3:150FBE9FB475C64B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:54113 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9344 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/init-core-data-001
   [junit4]   2> 101702 T157 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 101778 T157 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 101792 T158 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 102196 T157 oasc.ZkTestServer.run start zk server on port:54100
   [junit4]   2> 102393 T157 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 102703 T157 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 102978 T165 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4dcbbdda name:ZooKeeperConnection Watcher:127.0.0.1:54100 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 102978 T157 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 102979 T157 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 102998 T157 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 103192 T157 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 103193 T157 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 103198 T168 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6e891f79 name:ZooKeeperConnection Watcher:127.0.0.1:54100/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 103199 T157 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 103200 T157 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 103227 T157 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 103232 T157 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 103236 T157 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 103239 T157 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 103243 T157 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 103244 T157 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 103250 T157 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 103251 T157 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 103256 T157 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 103256 T157 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 103261 T157 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 103261 T157 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 103265 T157 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 103265 T157 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 103269 T157 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 103270 T157 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 103300 T157 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 103301 T157 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 103305 T157 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 103306 T157 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 103309 T157 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 103310 T157 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 103314 T157 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 103314 T157 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 103318 T157 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 103318 T157 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 105888 T157 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores/collection1
   [junit4]   2> 105895 T157 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 105899 T157 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:54106
   [junit4]   2> 105899 T157 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/tempDir-001/control/data, hostContext=/, hostPort=54105}
   [junit4]   2> 105908 T157 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 105908 T157 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/'
   [junit4]   2> 105978 T157 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/solr.xml
   [junit4]   2> 106074 T157 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores
   [junit4]   2> 106074 T157 oasc.CoreContainer.<init> New CoreContainer 470219515
   [junit4]   2> 106074 T157 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/]
   [junit4]   2> 106075 T157 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/lib
   [junit4]   2> 106076 T157 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/lib).
   [junit4]   2> 106089 T157 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 106089 T157 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 106089 T157 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 106090 T157 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 106090 T157 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 106090 T157 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 106090 T157 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 106091 T157 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 106091 T157 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 106091 T157 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 106092 T157 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 106099 T157 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 106100 T157 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 106100 T157 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 106101 T157 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 106101 T157 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54100/solr
   [junit4]   2> 106158 T157 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 106159 T157 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 106160 T157 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 106162 T180 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@220fe47c name:ZooKeeperConnection Watcher:127.0.0.1:54100 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 106163 T157 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 106163 T157 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 106184 T157 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 106186 T183 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3f2d3e9d name:ZooKeeperConnection Watcher:127.0.0.1:54100/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 106186 T157 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 106211 T157 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 106229 T157 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 106239 T157 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 106244 T157 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 106248 T157 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 106287 T157 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 106290 T157 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 106293 T157 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 106295 T157 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54105_
   [junit4]   2> 106314 T157 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54105_
   [junit4]   2> 106318 T157 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 106322 T157 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 106325 T157 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 106364 T157 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:54105_
   [junit4]   2> 106365 T157 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 106368 T157 oasc.Overseer.start Overseer (id=93610334489739267-127.0.0.1:54105_-n_0000000000) starting
   [junit4]   2> 106410 T157 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 106473 T157 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 106512 T185 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 106512 T157 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 106546 T184 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 106576 T157 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores
   [junit4]   2> 106607 T157 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {schema=schema.xml, coreNodeName=, shard=, name=collection1, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores/collection1, loadOnStartup=true, dataDir=data/, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores/collection1/, collection=control_collection}
   [junit4]   2> 106607 T157 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores/collection1/
   [junit4]   2> 106608 T157 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 106609 T187 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 106610 T187 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 106633 T183 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 106633 T187 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 106642 T184 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:54105_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54105",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null} current state version: 0
   [junit4]   2> 106722 T184 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:54105_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54105",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 106733 T184 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 106816 T184 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 106843 T183 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 107634 T187 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 107634 T187 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 107635 T187 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 107635 T187 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 107637 T187 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 107637 T187 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores/collection1/'
   [junit4]   2> 107677 T187 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 107698 T187 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 107728 T187 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 107752 T187 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 107797 T187 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 107799 T187 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 107817 T187 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 108335 T187 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 108376 T187 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 108378 T187 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 108399 T187 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 108424 T187 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 108471 T187 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 108473 T187 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 108473 T187 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 108474 T187 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 108475 T187 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 108475 T187 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 108476 T187 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 108476 T187 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 108476 T187 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores/collection1/data/
   [junit4]   2> 108477 T187 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@44a0c74c
   [junit4]   2> 108488 T187 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores/collection1/data
   [junit4]   2> 108488 T187 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores/collection1/data/index/
   [junit4]   2> 108488 T187 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 108489 T187 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores/collection1/data/index
   [junit4]   2> 108489 T187 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=14, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 108490 T187 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1892ebd6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55b93b34),segFN=segments_1,generation=1}
   [junit4]   2> 108490 T187 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 108498 T187 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 108605 T187 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 108606 T187 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 108606 T187 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 108606 T187 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 108606 T187 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 108607 T187 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 108607 T187 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 108607 T187 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 108633 T187 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 108643 T187 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 108645 T187 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 108648 T187 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 108659 T187 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update,/admin/segments,standard,/admin/logging,/admin/plugins,/admin/system,/admin/file,/get,/update/json/docs,/admin/luke,/admin/mbeans,/admin/ping,/schema,/replication,/config,/update/json,/admin/properties,/update/csv,/admin/threads
   [junit4]   2> 108663 T187 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 108665 T187 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 108666 T187 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 108667 T187 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 108668 T187 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 108669 T187 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=25, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.792881320471249]
   [junit4]   2> 108669 T187 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1892ebd6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55b93b34),segFN=segments_1,generation=1}
   [junit4]   2> 108669 T187 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 108670 T187 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@658eb67b[collection1] main
   [junit4]   2> 108670 T187 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 108672 T187 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 108672 T187 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 108690 T187 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 108690 T187 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 108690 T187 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 108691 T187 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 108691 T187 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 108692 T187 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 108692 T187 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 108695 T188 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@658eb67b[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 108705 T187 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 108706 T191 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:54105 collection:control_collection shard:shard1
   [junit4]   2> 108707 T157 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 108708 T157 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 108758 T191 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 108768 T191 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 108771 T183 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 108772 T191 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 108772 T191 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 108773 T184 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C30 name=collection1 org.apache.solr.core.SolrCore@211f7810 url=http://127.0.0.1:54105/collection1 node=127.0.0.1:54105_ C30_STATE=coll:control_collection core:collection1 props:{core=collection1, state=down, node_name=127.0.0.1:54105_, base_url=http://127.0.0.1:54105}
   [junit4]   2> 108786 T191 C:control_collection S:shard1 C30 P54105 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:54105/collection1/
   [junit4]   2> 108787 T191 C:control_collection S:shard1 C30 P54105 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 108787 T191 C:control_collection S:shard1 C30 P54105 oasc.SyncStrategy.syncToMe http://127.0.0.1:54105/collection1/ has no replicas
   [junit4]   2> 108788 T191 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:54105/collection1/ shard1
   [junit4]   2> 108789 T183 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 108807 T157 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 108808 T191 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 108808 T157 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 108818 T195 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@40b5a9b2 name:ZooKeeperConnection Watcher:127.0.0.1:54100/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 108818 T157 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 108819 T157 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 108819 T157 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 108823 T183 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 108825 T184 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54105",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 108827 T157 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 108827 T157 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 108828 T157 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 108828 T157 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 108830 T198 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2f3e04c9 name:ZooKeeperConnection Watcher:127.0.0.1:54100/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 108831 T157 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 108831 T157 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 108834 T183 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 108836 T184 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 108836 T184 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 108837 T195 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 108837 T183 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 108875 T191 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:54105/collection1/ and leader is http://127.0.0.1:54105/collection1/
   [junit4]   2> 108876 T191 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:54105
   [junit4]   2> 108876 T191 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 108876 T191 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 108876 T191 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 108878 T183 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 108880 T184 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:54105_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54105",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "roles":null} current state version: 3
   [junit4]   2> 108881 T184 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:54105_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54105",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 108882 T184 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 108884 T183 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 108884 T195 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 108992 T195 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 108992 T183 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 110958 T157 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores/collection1
   [junit4]   2> 110962 T157 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001
   [junit4]   2> 110963 T157 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 110966 T157 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:54114
   [junit4]   2> 110967 T157 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=54113, solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/tempDir-001/jetty1}
   [junit4]   2> 110968 T157 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 110968 T157 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/'
   [junit4]   2> 111057 T157 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/solr.xml
   [junit4]   2> 111148 T157 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores
   [junit4]   2> 111148 T157 oasc.CoreContainer.<init> New CoreContainer 1308631172
   [junit4]   2> 111149 T157 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/]
   [junit4]   2> 111149 T157 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/lib
   [junit4]   2> 111160 T157 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/lib).
   [junit4]   2> 111175 T157 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 111176 T157 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 111176 T157 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 111176 T157 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 111177 T157 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 111177 T157 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 111178 T157 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 111178 T157 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 111178 T157 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 111179 T157 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 111179 T157 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 111191 T157 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 111192 T157 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 111193 T157 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 111193 T157 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 111193 T157 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54100/solr
   [junit4]   2> 111194 T157 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 111194 T157 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 111195 T157 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 111198 T210 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@45150b1b name:ZooKeeperConnection Watcher:127.0.0.1:54100 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 111198 T157 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 111199 T157 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 111203 T157 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 111205 T213 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@30aa7379 name:ZooKeeperConnection Watcher:127.0.0.1:54100/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 111205 T157 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 111215 T157 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 112227 T157 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54113_
   [junit4]   2> 112229 T157 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54113_
   [junit4]   2> 112234 T157 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 112243 T157 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores
   [junit4]   2> 112245 T157 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=collection1, schema=schema.xml, shard=, loadOnStartup=true, dataDir=data/, name=collection1, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores/collection1/, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores/collection1, transient=false}
   [junit4]   2> 112246 T157 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores/collection1/
   [junit4]   2> 112246 T157 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 112250 T214 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 112250 T214 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 112252 T183 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 112254 T214 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 112254 T214 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 112254 T184 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:54113_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54113",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "roles":null} current state version: 5
   [junit4]   2> 112255 T214 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 112255 T184 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:54113_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54113",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 112256 T184 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 112256 T184 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 112259 T184 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 112264 T214 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 112265 T214 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 112265 T214 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 112265 T214 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 112266 T214 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 112266 T214 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 112267 T214 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 112267 T214 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores/collection1/'
   [junit4]   2> 112285 T214 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 112289 T214 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 112300 T214 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 112338 T214 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 112388 T214 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 112390 T214 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 112401 T214 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 112910 T214 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 112927 T214 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 112929 T214 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 112948 T214 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 112957 T214 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 112979 T214 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 112980 T214 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 112980 T214 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 112981 T214 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 112982 T214 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 112982 T214 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 112982 T214 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 112982 T214 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 112983 T214 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 112983 T214 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@44a0c74c
   [junit4]   2> 112984 T214 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores/collection1/data
   [junit4]   2> 112984 T214 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 112984 T214 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 112985 T214 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 112985 T214 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=14, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 112986 T214 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@adbe555 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6aad3504),segFN=segments_1,generation=1}
   [junit4]   2> 112986 T214 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 112996 T214 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 112997 T214 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 112997 T214 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 112997 T214 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 112997 T214 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 112998 T214 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 112998 T214 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 112999 T214 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 112999 T214 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 113013 T214 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 113016 T214 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 113019 T214 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 113021 T214 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 113049 T214 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update,/admin/segments,standard,/admin/logging,/admin/plugins,/admin/system,/admin/file,/get,/update/json/docs,/admin/luke,/admin/mbeans,/admin/ping,/schema,/replication,/config,/update/json,/admin/properties,/update/csv,/admin/threads
   [junit4]   2> 113051 T214 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 113052 T214 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 113053 T214 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 113054 T214 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 113054 T214 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 113056 T214 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=25, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.792881320471249]
   [junit4]   2> 113056 T214 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@adbe555 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6aad3504),segFN=segments_1,generation=1}
   [junit4]   2> 113057 T214 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 113057 T214 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@578a1898[collection1] main
   [junit4]   2> 113057 T214 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 113059 T214 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 113059 T214 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 113060 T214 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 113060 T214 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 113060 T214 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 113061 T214 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 113061 T214 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 113061 T214 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 113062 T214 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 113072 T215 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@578a1898[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 113073 T214 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 113074 T218 C:collection1 S:shard3 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:54113 collection:collection1 shard:shard3
   [junit4]   2> 113075 T157 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 113075 T157 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 113075 T218 C:collection1 S:shard3 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
   [junit4]   2> 113083 T218 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard3
   [junit4]   2> 113086 T183 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 113087 T218 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 113087 T218 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C31 name=collection1 org.apache.solr.core.SolrCore@729f3241 url=http://127.0.0.1:54113/collection1 node=127.0.0.1:54113_ C31_STATE=coll:collection1 core:collection1 props:{core=collection1, state=down, node_name=127.0.0.1:54113_, base_url=http://127.0.0.1:54113}
   [junit4]   2> 113088 T218 C:collection1 S:shard3 C31 P54113 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:54113/collection1/
   [junit4]   2> 113088 T218 C:collection1 S:shard3 C31 P54113 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 113088 T184 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 113088 T218 C:collection1 S:shard3 C31 P54113 oasc.SyncStrategy.syncToMe http://127.0.0.1:54113/collection1/ has no replicas
   [junit4]   2> 113089 T218 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:54113/collection1/ shard3
   [junit4]   2> 113089 T218 C:collection1 S:shard3 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard3
   [junit4]   2> 113092 T184 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 113093 T213 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 113095 T213 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 113101 T184 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54113",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 113104 T184 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 113105 T213 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 113106 T213 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 113111 T183 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 113151 T218 C:collection1 S:shard3 oasc.ZkController.register We are http://127.0.0.1:54113/collection1/ and leader is http://127.0.0.1:54113/collection1/
   [junit4]   2> 113152 T218 C:collection1 S:shard3 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:54113
   [junit4]   2> 113153 T218 C:collection1 S:shard3 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 113153 T218 C:collection1 S:shard3 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 113153 T218 C:collection1 S:shard3 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 113156 T183 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 113159 T184 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:54113_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54113",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "roles":null} current state version: 5
   [junit4]   2> 113160 T184 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:54113_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54113",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "roles":null}
   [junit4] 

[...truncated too long message...]

0406174556825
   [junit4]   2> 1762047 T3781 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1762049 T3670 C:control_collection S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/fs,null}
   [junit4]   2> 1762225 T3670 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:34962 34962
   [junit4]   2> 1762569 T3671 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:34962 34962
   [junit4]   2> 1762570 T3671 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 		2	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1762571 T3670 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:34979/fs
   [junit4]   2> 1762571 T3670 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:34969/fs
   [junit4]   2> 1762571 T3670 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:34986/fs
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=9D5B81451A89ABB3 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=es_ES -Dtests.timezone=Pacific/Midway -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   71.9s J0 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:34979 failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([9D5B81451A89ABB3:150FBE9FB475C64B]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:34979 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1762587 T3670 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 9D5B81451A89ABB3-001
   [junit4]   2> 71897 T3669 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Lucene50, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=es_ES, timezone=Pacific/Midway
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=185934920,total=312475648
   [junit4]   2> NOTE: All tests run in this JVM: [TestUniqueKeyFieldResource, OutOfBoxZkACLAndCredentialsProvidersTest, SystemInfoHandlerTest, TestReplicaProperties, ZkStateWriterTest, TestHdfsUpdateLog, TestBinaryResponseWriter, TestWriterPerf, FastVectorHighlighterTest, TestLRUCache, TestStressVersions, TestAddFieldRealTimeGet, PrimUtilsTest, MinimalSchemaTest, OverseerCollectionProcessorTest, SuggesterFSTTest, BlockCacheTest, MBeansHandlerTest, PrimitiveFieldTypeTest, TestManagedResource, TestRecovery, HttpPartitionTest, RequestHandlersTest, BadComponentTest, TestJettySolrRunner, CachingDirectoryFactoryTest, TestCloudPivotFacet, CustomCollectionTest, TestNRTOpen, OutputWriterTest, TestConfigSets, TestArbitraryIndexDir, TestFastOutputStream, RecoveryZkTest, TestHighFrequencyDictionaryFactory, BinaryUpdateRequestHandlerTest, TestManagedSchemaFieldTypeResource, TestExactSharedStatsCache, BasicZkTest, TestSort, SolrTestCaseJ4Test, TestDocBasedVersionConstraints, TestSolrQueryParserResource, DefaultValueUpdateProcessorTest, DirectSolrSpellCheckerTest, DeleteLastCustomShardedReplicaTest, ClusterStateUpdateTest, TestRangeQuery, TestExpandComponent, MergeStrategyTest, TestLMDirichletSimilarityFactory, TestSolrDynamicMBean, TestConfig, SimplePostToolTest, TestBulkSchemaConcurrent, TestIntervalFaceting, DistributedQueryComponentOptimizationTest, TestMacros, HdfsUnloadDistributedZkTest, FullSolrCloudDistribCmdsTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J0 in 72.68s, 1 test, 1 error <<< FAILURES!

[...truncated 723 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 479 suites, 1903 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 60 minutes 57 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 227544524 bytes
Compression is 0.0%
Took 45 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2902 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2902/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:62732/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:62732/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([26BC33507B4A9FC1:AEE80C8AD5B6F239]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:62732 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:63007 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:63007 failed to respond
	at __randomizedtesting.SeedInfo.seed([26BC33507B4A9FC1:AEE80C8AD5B6F239]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:63007 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:62804 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:62804 failed to respond
	at __randomizedtesting.SeedInfo.seed([26BC33507B4A9FC1:AEE80C8AD5B6F239]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:62804 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 10002 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/init-core-data-001
   [junit4]   2> 1486847 T4980 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 1486852 T4980 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1486852 T4981 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1486952 T4980 oasc.ZkTestServer.run start zk server on port:62708
   [junit4]   2> 1486953 T4980 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1486953 T4980 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1486957 T4988 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@37840492 name:ZooKeeperConnection Watcher:127.0.0.1:62708 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1486957 T4980 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1486957 T4980 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1486957 T4980 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1486959 T4980 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1486960 T4980 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1486961 T4991 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@e4dd35c name:ZooKeeperConnection Watcher:127.0.0.1:62708/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1486962 T4980 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1486962 T4980 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1486962 T4980 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1486964 T4980 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1486965 T4980 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1486966 T4980 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1486967 T4980 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1486968 T4980 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1486969 T4980 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1486970 T4980 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1487073 T4980 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1487073 T4980 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1487075 T4980 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1487075 T4980 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1487077 T4980 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1487077 T4980 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1487079 T4980 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1487079 T4980 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1487081 T4980 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1487081 T4980 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1487083 T4980 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1487083 T4980 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1487085 T4980 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1487085 T4980 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1487087 T4980 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1487087 T4980 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1487089 T4980 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1487089 T4980 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1488674 T4980 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores/collection1
   [junit4]   2> 1488678 T4980 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1488682 T4980 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:62717
   [junit4]   2> 1488683 T4980 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, hostPort=62716, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/tempDir-001/control/data}
   [junit4]   2> 1488683 T4980 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 1488683 T4980 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/'
   [junit4]   2> 1488727 T4980 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/solr.xml
   [junit4]   2> 1488781 T4980 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores
   [junit4]   2> 1488782 T4980 oasc.CoreContainer.<init> New CoreContainer 1635486708
   [junit4]   2> 1488782 T4980 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/]
   [junit4]   2> 1488782 T4980 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/lib
   [junit4]   2> 1488783 T4980 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/lib).
   [junit4]   2> 1488792 T4980 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1488793 T4980 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1488793 T4980 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1488793 T4980 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1488793 T4980 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1488794 T4980 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1488794 T4980 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1488794 T4980 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1488794 T4980 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1488794 T4980 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1488795 T4980 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1488802 T4980 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1488803 T4980 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1488803 T4980 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1488803 T4980 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1488804 T4980 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:62708/solr
   [junit4]   2> 1488804 T4980 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1488804 T4980 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1488805 T4980 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1488806 T5003 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7be47528 name:ZooKeeperConnection Watcher:127.0.0.1:62708 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1488807 T4980 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1488807 T4980 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1488809 T4980 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1488810 T5006 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7c304ccf name:ZooKeeperConnection Watcher:127.0.0.1:62708/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1488810 T4980 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1488811 T4980 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1488813 T4980 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1488814 T4980 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1488815 T4980 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1488817 T4980 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1488818 T4980 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1488819 T4980 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1488820 T4980 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1488821 T4980 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62716_
   [junit4]   2> 1488821 T4980 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:62716_
   [junit4]   2> 1488822 T4980 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1488823 T4980 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1488824 T4980 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1488825 T4980 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:62716_
   [junit4]   2> 1488825 T4980 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1488826 T4980 oasc.Overseer.start Overseer (id=93609472381485059-127.0.0.1:62716_-n_0000000000) starting
   [junit4]   2> 1488827 T4980 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1488831 T4980 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1488832 T5008 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1488832 T4980 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1488832 T5007 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1488835 T4980 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores
   [junit4]   2> 1488837 T4980 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores/collection1, collection=control_collection, schema=schema.xml, coreNodeName=, shard=, loadOnStartup=true, config=solrconfig.xml, dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores/collection1/}
   [junit4]   2> 1488837 T4980 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores/collection1/
   [junit4]   2> 1488837 T4980 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1488838 T5010 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1488838 T5010 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1488839 T5010 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1488839 T5006 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1488839 T5007 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62716",
   [junit4]   2> 	  "node_name":"127.0.0.1:62716_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null} current state version: 0
   [junit4]   2> 1488840 T5007 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62716",
   [junit4]   2> 	  "node_name":"127.0.0.1:62716_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 1488840 T5007 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1488840 T5007 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1488841 T5006 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1489839 T5010 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1489840 T5010 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1489841 T5010 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1489841 T5010 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1489841 T5010 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1489842 T5010 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores/collection1/'
   [junit4]   2> 1489860 T5010 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1489862 T5010 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1489873 T5010 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1489897 T5010 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1489950 T5010 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1490052 T5010 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1490063 T5010 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1490500 T5010 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1490516 T5010 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1490519 T5010 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1490528 T5010 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1490533 T5010 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1490537 T5010 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1490537 T5010 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1490538 T5010 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1490538 T5010 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1490539 T5010 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1490539 T5010 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1490539 T5010 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1490539 T5010 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1490540 T5010 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores/collection1/data/
   [junit4]   2> 1490540 T5010 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@78e9fb08
   [junit4]   2> 1490541 T5010 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores/collection1/data
   [junit4]   2> 1490541 T5010 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores/collection1/data/index/
   [junit4]   2> 1490541 T5010 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1490542 T5010 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores/collection1/data/index
   [junit4]   2> 1490542 T5010 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=36, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1490543 T5010 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@cbce229 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@44a3f847),segFN=segments_1,generation=1}
   [junit4]   2> 1490543 T5010 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1490549 T5010 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1490549 T5010 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1490550 T5010 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1490550 T5010 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1490550 T5010 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1490550 T5010 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1490551 T5010 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1490551 T5010 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1490552 T5010 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1490574 T5010 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1490577 T5010 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1490579 T5010 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1490581 T5010 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1490587 T5010 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: standard,/replication,/admin/system,/admin/properties,/update/json,/admin/file,/update,/admin/logging,/config,/update/json/docs,/admin/plugins,/admin/ping,/admin/threads,/admin/mbeans,/admin/luke,/get,/admin/segments,/schema,/update/csv
   [junit4]   2> 1490589 T5010 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1490590 T5010 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1490591 T5010 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1490592 T5010 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1490592 T5010 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1490593 T5010 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=27, maxMergedSegmentMB=92.4228515625, floorSegmentMB=0.3427734375, forceMergeDeletesPctAllowed=1.1550384131963098, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.49791742044625653
   [junit4]   2> 1490594 T5010 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@cbce229 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@44a3f847),segFN=segments_1,generation=1}
   [junit4]   2> 1490594 T5010 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1490594 T5010 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@460fdd6d[collection1] main
   [junit4]   2> 1490595 T5010 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1490595 T5010 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1490596 T5010 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1490596 T5010 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1490596 T5010 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1490596 T5010 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1490597 T5010 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1490597 T5010 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1490597 T5010 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1490598 T5010 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1490598 T5011 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@460fdd6d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1490599 T5010 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1490600 T5014 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:62716 collection:control_collection shard:shard1
   [junit4]   2> 1490600 T4980 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 1490600 T4980 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1490600 T5014 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1490602 T4980 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1490603 T4980 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1490604 T5014 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1490604 T5018 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@44539bd3 name:ZooKeeperConnection Watcher:127.0.0.1:62708/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1490605 T4980 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1490605 T4980 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1490605 T4980 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1490605 T5006 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1490606 T5014 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1490606 T5014 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C660 name=collection1 org.apache.solr.core.SolrCore@4ecb64b6 url=http://127.0.0.1:62716/collection1 node=127.0.0.1:62716_ C660_STATE=coll:control_collection core:collection1 props:{core=collection1, base_url=http://127.0.0.1:62716, node_name=127.0.0.1:62716_, state=down}
   [junit4]   2> 1490607 T5014 C:control_collection S:shard1 C660 P62716 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:62716/collection1/
   [junit4]   2> 1490607 T5007 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 1490607 T5014 C:control_collection S:shard1 C660 P62716 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1490607 T5014 C:control_collection S:shard1 C660 P62716 oasc.SyncStrategy.syncToMe http://127.0.0.1:62716/collection1/ has no replicas
   [junit4]   2> 1490607 T5014 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:62716/collection1/ shard1
   [junit4]   2> 1490608 T5014 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1490608 T4980 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1490608 T5006 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1490608 T4980 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 1490608 T5018 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1490609 T4980 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1490610 T4980 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1490611 T5021 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2b9870e0 name:ZooKeeperConnection Watcher:127.0.0.1:62708/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1490612 T4980 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1490612 T5006 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1490612 T4980 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1490613 T5007 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62716",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 1490615 T5007 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 1490615 T5007 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 1490616 T5018 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1490616 T5006 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1490618 T5006 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1490618 T5007 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 1490619 T5018 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1490619 T5006 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1490663 T5014 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:62716/collection1/ and leader is http://127.0.0.1:62716/collection1/
   [junit4]   2> 1490664 T5014 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:62716
   [junit4]   2> 1490664 T5014 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1490664 T5014 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1490664 T5014 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1490666 T5006 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1490667 T5007 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62716",
   [junit4]   2> 	  "node_name":"127.0.0.1:62716_",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1"} current state version: 4
   [junit4]   2> 1490667 T5007 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62716",
   [junit4]   2> 	  "node_name":"127.0.0.1:62716_",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 1490771 T5006 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1490771 T5018 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1492134 T4980 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores/collection1
   [junit4]   2> 1492136 T4980 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001
   [junit4]   2> 1492137 T4980 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1492139 T4980 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:62733
   [junit4]   2> 1492139 T4980 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=62732, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores}
   [junit4]   2> 1492140 T4980 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 1492140 T4980 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/'
   [junit4]   2> 1492176 T4980 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/solr.xml
   [junit4]   2> 1492219 T4980 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores
   [junit4]   2> 1492220 T4980 oasc.CoreContainer.<init> New CoreContainer 125063838
   [junit4]   2> 1492220 T4980 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/]
   [junit4]   2> 1492220 T4980 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/lib
   [junit4]   2> 1492221 T4980 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/lib).
   [junit4]   2> 1492231 T4980 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1492231 T4980 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1492231 T4980 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1492231 T4980 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1492232 T4980 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1492232 T4980 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1492232 T4980 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1492232 T4980 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1492232 T4980 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1492233 T4980 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1492233 T4980 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1492238 T4980 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1492239 T4980 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1492239 T4980 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1492240 T4980 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1492240 T4980 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:62708/solr
   [junit4]   2> 1492240 T4980 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1492240 T4980 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1492241 T4980 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1492243 T5033 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@872ac36 name:ZooKeeperConnection Watcher:127.0.0.1:62708 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1492244 T4980 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1492244 T4980 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1492247 T4980 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1492248 T5036 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@884ee6c name:ZooKeeperConnection Watcher:127.0.0.1:62708/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1492248 T4980 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1492252 T4980 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1493256 T4980 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62732_
   [junit4]   2> 1493257 T4980 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:62732_
   [junit4]   2> 1493260 T4980 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1493261 T4980 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores
   [junit4]   2> 1493264 T4980 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, coreNodeName=, transient=false, schema=schema.xml, name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores/collection1/, shard=, dataDir=data/, loadOnStartup=true, collection=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores/collection1}
   [junit4]   2> 1493264 T4980 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores/collection1/
   [junit4]   2> 1493264 T4980 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1493266 T5037 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1493266 T5037 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1493267 T5006 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1493267 T5037 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 1493267 T5037 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 1493267 T5007 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62732",
   [junit4]   2> 	  "node_name":"127.0.0.1:62732_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null} current state version: 5
   [junit4]   2> 1493268 T5037 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 1493268 T5007 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62732",
   [junit4]   2> 	  "node_name":"127.0.0.1:62732_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 1493268 T5007 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 1493268 T5007 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1493268 T5037 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 1493268 T5037 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1493269 T5007 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 1493269 T5036 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1493270 T5036 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 1494269 T5037 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1494270 T5037 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1494270 T5037 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1494270 T5037 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1494271 T5037 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1494271 T5037 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores/collection1/'
   [junit4]   2> 1494284 T5037 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1494286 T5037 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1494294 T5037 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1494324 T5037 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1494362 T5037 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1494464 T5037 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1494472 T5037 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1494762 T5037 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1494774 T5037 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1494776 T5037 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1494784 T5037 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1494787 T5037 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1494790 T5037 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1494791 T5037 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1494791 T5037 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1494791 T5037 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1494792 T5037 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1494792 T5037 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1494792 T5037 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1494793 T5037 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1494793 T5037 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 1494793 T5037 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@78e9fb08
   [junit4]   2> 1494794 T5037 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores/collection1/data
   [junit4]   2> 1494794 T5037 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 1494794 T5037 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1494795 T5037 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 1494795 T5037 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=36, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1494796 T5037 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4d8423ef lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1e5347ab),segFN=segments_1,generation=1}
   [junit4]   2> 1494796 T5037 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1494801 T5037 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1494801 T5037 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1494802 T5037 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1494802 T5037 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1494802 T5037 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1494802 T5037 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1494802 T5037 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1494803 T5037 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1494803 T5037 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1494822 T5037 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1494824 T5037 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1494826 T5037 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1494827 T5037 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1494834 T5037 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: standard,/replication,/admin/system,/admin/properties,/update/json,/admin/file,/update,/admin/logging,/config,/update/json/docs,/admin/plugins,/admin/ping,/admin/threads,/admin/mbeans,/admin/luke,/get,/admin/segments,/schema,/update/csv
   [junit4]   2> 1494836 T5037 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1494838 T5037 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1494838 T5037 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1494839 T5037 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1494839 T5037 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1494840 T5037 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=27, maxMergedSegmentMB=92.4228515625, floorSegmentMB=0.3427734375, forceMergeDeletesPctAllowed=1.1550384131963098, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.49791742044625653
   [junit4]   2> 1494841 T5037 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4d8423ef lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1e5347ab),segFN=segments_1,generation=1}
   [junit4]   2> 1494841 T5037 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1494841 T5037 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@56a3f1e1[collection1] main
   [junit4]   2> 1494842 T5037 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1494842 T5037 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1494843 T5037 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1494843 T5037 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1494843 T5037 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1494843 T5037 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1494844 T5037 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1494844 T5037 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1494844 T5037 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1494844 T5037 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1494845 T5038 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@56a3f1e1[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1494845 T5037 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1494846 T5041 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:62732 collection:collection1 shard:shard1
   [junit4]   2> 1494846 T4980 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 1494847 T4980 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1494847 T5041 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1494850 T5041 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1494852 T5006 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1494852 T5041 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1494852 T5041 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C661 name=collection1 org.apache.solr.core.SolrCore@55daa5ce url=http://127.0.0.1:62732/collection1 node=127.0.0.1:62732_ C661_STATE=coll:collection1 core:collection1 props:{core=collection1, base_url=http://127.0.0.1:62732, node_name=127.0.0.1:62732_, state=down}
   [junit4]   2> 1494852 T5041 C:collection1 S:shard1 C661 P62732 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:62732/collection1/
   [junit4]   2> 1494852 T5007 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 1494853 T5041 C:collection1 S:shard1 C661 P62732 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1494853 T5041 C:collection1 S:shard1 C661 P62732 oasc.SyncStrategy.syncToMe http://127.0.0.1:62732/collection1/ has no replicas
   [junit4]   2> 1494853 T5041 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:62732/collection1/ shard1
   [junit4]   2> 1494853 T5041 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1494853 T5007 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 1494854 T5036 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1494855 T5036 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 1494856 T5006 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1494857 T5007 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62732",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 1494858 T5007 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 1494858 T5036 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1494859 T5036 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 1494907 T5041 C:collection1 S:shard1

[...truncated too long message...]

lk to ZK
   [junit4]   2> 1633632 T3227 C:control_collection S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 1633736 T3227 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:62998 62998
   [junit4]   2> 1634082 T3228 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:62998 62998
   [junit4]   2> 1634085 T3228 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 		2	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1634085 T3227 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:63011/
   [junit4]   2> 1634086 T3227 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 6 connections to: http://127.0.0.1:63007/
   [junit4]   2> 1634086 T3227 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:63001/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=26BC33507B4A9FC1 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=fr_BE -Dtests.timezone=America/Fortaleza -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   71.9s J0 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:63007 failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([26BC33507B4A9FC1:AEE80C8AD5B6F239]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:63007 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1634102 T3227 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 26BC33507B4A9FC1-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {timestamp=Lucene50(blocksize=128), a_t=Lucene50(blocksize=128), multiDefault=Lucene50(blocksize=128), id=PostingsFormat(name=LuceneFixedGap), _version_=FSTOrd50, range_facet_l=Lucene50(blocksize=128), intDefault=FSTOrd50, text=FST50}, docValues:{timestamp=DocValuesFormat(name=Direct)}, sim=DefaultSimilarity, locale=fr_BE, timezone=America/Fortaleza
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=97899896,total=327680000
   [junit4]   2> NOTE: All tests run in this JVM: [TestLRUStatsCache, DateRangeFieldTest, LukeRequestHandlerTest, CacheHeaderTest, TestCloudInspectUtil, TestCharFilters, WordBreakSolrSpellCheckerTest, TestJsonRequest, TestPostingsSolrHighlighter, TestBinaryField, TestQueryUtils, TestFieldTypeCollectionResource, TestConfigOverlay, TestFuzzyAnalyzedSuggestions, MultiThreadedOCPTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, DistanceFunctionTest, TestConfig, BlockDirectoryTest, SimpleCollectionCreateDeleteTest, PreAnalyzedUpdateProcessorTest, CursorMarkTest, TestMiniSolrCloudCluster, DeleteLastCustomShardedReplicaTest, AnalyticsQueryTest, SolrInfoMBeanTest, TestCloudManagedSchema, TestDynamicFieldResource, TestIntervalFaceting, AssignTest, HdfsDirectoryFactoryTest, CustomCollectionTest, TestLMDirichletSimilarityFactory, TestLFUCache, DocExpirationUpdateProcessorFactoryTest, HttpPartitionTest, SharedFSAutoReplicaFailoverUtilsTest, ChangedSchemaMergeTest, TestAnalyzedSuggestions, VMParamsZkACLAndCredentialsProvidersTest, TestPHPSerializedResponseWriter, TestFieldCollectionResource, TestInitQParser, HdfsBasicDistributedZkTest, TestZkChroot, CursorPagingTest, RemoteQueryErrorTest, ReturnFieldsTest, ScriptEngineTest, TestCodecSupport, DistributedSuggestComponentTest, TestShardHandlerFactory, BadComponentTest, UpdateRequestProcessorFactoryTest, TestReqParamsAPI, TestImplicitCoreProperties, TestHashPartitioner, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J0 in 71.97s, 1 test, 1 error <<< FAILURES!

[...truncated 722 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 479 suites, 1903 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 54 minutes 45 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 228365171 bytes
Compression is 0.0%
Took 44 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2901 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2901/

4 tests failed.
REGRESSION:  org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds

Error Message:
soft529 wasn't fast enough

Stack Trace:
java.lang.AssertionError: soft529 wasn't fast enough
	at __randomizedtesting.SeedInfo.seed([E74DB75054A0944:5FA022F5B43939E3]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNotNull(Assert.java:526)
	at org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds(SoftAutoCommitTest.java:111)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:25435/_f/em/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:25435/_f/em/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([E74DB75054A0944:8620E4AFABB664BC]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:25435 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:41448 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:41448 failed to respond
	at __randomizedtesting.SeedInfo.seed([E74DB75054A0944:8620E4AFABB664BC]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:41448 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:43336 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:43336 failed to respond
	at __randomizedtesting.SeedInfo.seed([E74DB75054A0944:8620E4AFABB664BC]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:43336 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9228 lines...]
   [junit4] Suite: org.apache.solr.update.SoftAutoCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001
   [junit4]   2> 15934 T16 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 15936 T16 oas.SolrTestCaseJ4.initCore ####initCore
   [junit4]   2> 15936 T16 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 15938 T16 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 15939 T16 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 15940 T16 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 16107 T16 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 16199 T16 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 16345 T16 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 16347 T16 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 16454 T16 oass.IndexSchema.readSchema [null] Schema name=test
   [junit4]   2> 19050 T16 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 19180 T16 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 19190 T16 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 20464 T16 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 20491 T16 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 20497 T16 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 20503 T16 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 20504 T16 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 20504 T16 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 20505 T16 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 20506 T16 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 20519 T16 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 20520 T16 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr
   [junit4]   2> 20520 T16 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/'
   [junit4]   2> 20562 T16 oasc.CoreContainer.<init> New CoreContainer 1713272514
   [junit4]   2> 20563 T16 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/]
   [junit4]   2> 20563 T16 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/lib
   [junit4]   2> 20564 T16 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/lib).
   [junit4]   2> 20581 T16 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 600000
   [junit4]   2> 20582 T16 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 20582 T16 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 60000
   [junit4]   2> 20583 T16 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 20583 T16 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 20584 T16 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 20584 T16 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 20585 T16 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 20585 T16 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 20586 T16 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 20586 T16 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 20595 T16 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=30000&connTimeout=30000&retry=true
   [junit4]   2> 20597 T16 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 20598 T16 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 20598 T16 oasc.CoreContainer.load Node Name: testNode
   [junit4]   2> 20601 T16 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, loadOnStartup=true, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, transient=false, collection=collection1, schema=schema.xml, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001, instanceDir=collection1, shard=shard1}
   [junit4]   2> 20602 T17 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 20604 T17 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 20605 T17 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 20605 T17 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 20720 T17 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 20756 T17 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 20856 T17 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 20858 T17 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 20956 T17 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 21430 T17 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 21456 T17 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 21459 T17 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 21485 T17 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 21491 T17 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 21495 T17 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 21496 T17 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 21497 T17 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 21497 T17 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 21498 T17 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 21498 T17 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 21499 T17 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 21499 T17 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 21500 T17 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001/
   [junit4]   2> 21508 T17 c:collection1 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 21508 T17 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001
   [junit4]   2> 21509 T17 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001/index/
   [junit4]   2> 21509 T17 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001/index' doesn't exist. Creating new index...
   [junit4]   2> 21510 T17 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001/index
   [junit4]   2> 21511 T17 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=10, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 21519 T17 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7d97bf8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a07f17),segFN=segments_1,generation=1}
   [junit4]   2> 21520 T17 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 21856 T17 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 21973 T17 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 21981 T17 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 21982 T17 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 21982 T17 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 21983 T17 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 21983 T17 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 22017 T17 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 22018 T17 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 22335 T17 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 22336 T17 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 22337 T17 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 22374 T17 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 22378 T17 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 22381 T17 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 22384 T17 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 22689 T17 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 22815 T17 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/ping,/update/json,/replication,spellCheckCompRH_Direct,/get,mock,/search-facet-invariants,mltrh,spellCheckWithWordbreak_Direct,/admin/plugins,/update/csv,/admin/file,/admin/mbeans,dismax,/search-facet-def,spellCheckWithWordbreak,defaults,/admin/segments,/admin/threads,/config,tvrh,/admin/properties,standard,/debug/dump,/update/json/docs,lazy,/mlt,spellCheckCompRH1,/schema,/terms,spellCheckCompRH,/admin/system,/admin/logging,/update,/admin/luke
   [junit4]   2> 22827 T17 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 22831 T17 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 22965 T17 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 22966 T17 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 23032 T17 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 23033 T17 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 23034 T17 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=11, maxMergedSegmentMB=98.2822265625, floorSegmentMB=1.3701171875, forceMergeDeletesPctAllowed=26.756904425149433, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8147500519206919
   [junit4]   2> 23035 T17 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7d97bf8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a07f17),segFN=segments_1,generation=1}
   [junit4]   2> 23036 T17 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 23037 T17 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@785e5b21[collection1] main
   [junit4]   2> 23056 T17 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 23057 T17 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 23058 T17 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 23059 T17 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 23542 T17 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 24040 T17 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 24041 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 24045 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 24045 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 24046 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 24049 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 24052 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 24054 T17 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 24054 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 24056 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 24058 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 24059 T18 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@785e5b21[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 24092 T16 oas.SolrTestCaseJ4.initCore ####initCore end
   [junit4]   2> 24107 T16 oas.SolrTestCaseJ4.setUp ###Starting testSoftAndHardCommitMaxTimeMixedAdds
   [junit4]   2> 24108 T16 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 24111 T16 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 24112 T16 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 24113 T16 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 24359 T16 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 24424 T16 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 24554 T16 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 24555 T16 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 24619 T16 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 25072 T16 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 25097 T16 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 25100 T16 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 25126 T16 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 25139 T16 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 25144 T16 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 25146 T16 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 25146 T16 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 25147 T16 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 25148 T16 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 25148 T16 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 25149 T16 oasc.CoreContainer.reload Reloading SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 25150 T16 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001/
   [junit4]   2> 25150 T16 c:collection1 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 25151 T16 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001/index/
   [junit4]   2> 25155 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 25156 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 25156 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 25157 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 25157 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 25158 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 25158 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 25159 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 25159 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 25160 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 25161 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 25161 T16 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 25192 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 25194 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 25196 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 25199 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 25202 T16 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 25204 T16 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/ping,/update/json,/replication,spellCheckCompRH_Direct,/get,mock,/search-facet-invariants,mltrh,spellCheckWithWordbreak_Direct,/admin/plugins,/update/csv,/admin/file,/admin/mbeans,dismax,/search-facet-def,spellCheckWithWordbreak,defaults,/admin/segments,/admin/threads,/config,tvrh,/admin/properties,standard,/debug/dump,/update/json/docs,lazy,/mlt,spellCheckCompRH1,/schema,/terms,spellCheckCompRH,/admin/system,/admin/logging,/update,/admin/luke
   [junit4]   2> 25206 T16 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 25209 T16 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 25217 T16 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 25218 T16 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 25219 T16 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@5ccd6f35[collection1] main
   [junit4]   2> 25219 T16 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 25220 T16 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 25220 T16 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 25221 T16 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 25224 T16 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 25239 T16 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 25240 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 25240 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
   [junit4]   2> 25241 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
   [junit4]   2> 25241 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Closing old IndexWriter... core=collection1
   [junit4]   2> 25241 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 25242 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 25243 T16 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=15, maxMergeAtOnceExplicit=42, maxMergedSegmentMB=76.8359375, floorSegmentMB=1.505859375, forceMergeDeletesPctAllowed=23.071419198270572, segmentsPerTier=37.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 25243 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 25244 T16 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7d97bf8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a07f17),segFN=segments_1,generation=1}
   [junit4]   2> 25245 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 25245 T16 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 25246 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
   [junit4]   2> 25246 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 25246 T16 c:collection1 oasc.SolrCore.getSearcher WARN [collection1] PERFORMANCE WARNING: Overlapping onDeckSearchers=2
   [junit4]   2> 25247 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 25247 T16 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@760d62e0[collection1] main
   [junit4]   2> 25248 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 25248 T16 c:collection1 oasc.CoreContainer.registerCore replacing core: collection1
   [junit4]   2> 25249 T16 c:collection1 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@744e4cf6
   [junit4]   2> 25250 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 25250 T16 c:collection1 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=0,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=0,transaction_logs_total_number=0}
   [junit4]   2> 25250 T20 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5ccd6f35[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 25251 T16 c:collection1 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 25251 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 25253 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 25253 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 25253 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 25255 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 25256 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 25257 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 25258 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 25259 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 25260 T20 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@760d62e0[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> ASYNC  NEW_CORE C0 name=collection1 org.apache.solr.core.SolrCore@58169f1b
   [junit4]   2> 26863 T16 c:collection1 C0 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[529 (1497736543998050304)]} 0 1594
   [junit4]   2> 27353 T22 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 27945 T16 c:collection1 oas.SolrTestCaseJ4.tearDown ###Ending testSoftAndHardCommitMaxTimeMixedAdds
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SoftAutoCommitTest -Dtests.method=testSoftAndHardCommitMaxTimeMixedAdds -Dtests.seed=E74DB75054A0944 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=sr_BA_#Latn -Dtests.timezone=America/Merida -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE 3.92s J2 | SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: soft529 wasn't fast enough
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([E74DB75054A0944:5FA022F5B43939E3]:0)
   [junit4]    > 	at org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds(SoftAutoCommitTest.java:111)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 28025 T16 c:collection1 oas.SolrTestCaseJ4.setUp ###Starting testSoftAndHardCommitMaxTimeDelete
   [junit4]   2> 28026 T16 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 28027 T16 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 28028 T16 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 28029 T16 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 28053 T21 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 28187 T16 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 28231 T16 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 28350 T16 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 28353 T16 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 28361 T22 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@6ec3e67f[collection1] main
   [junit4]   2> 28410 T22 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 28411 T20 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6ec3e67f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):C1)))}
   [junit4]   2> 28413 T21 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7d97bf8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a07f17),segFN=segments_1,generation=1}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7d97bf8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a07f17),segFN=segments_2,generation=2}
   [junit4]   2> 28413 T21 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 28415 T21 c:collection1 oasc.SolrCore.openNewSearcher SolrIndexSearcher has not changed - not re-opening: org.apache.solr.search.SolrIndexSearcher
   [junit4]   2> 28416 T21 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 28513 T16 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 28954 T16 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 28979 T16 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 28982 T16 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 29008 T16 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 29014 T16 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 29019 T16 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 29020 T16 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 29021 T16 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 29021 T16 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 29022 T16 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 29022 T16 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 29023 T16 c:collection1 oasc.CoreContainer.reload Reloading SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 29024 T16 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001/
   [junit4]   2> 29024 T16 c:collection1 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 29025 T16 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001/index/
   [junit4]   2> 29028 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 29029 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 29029 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 29030 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 29030 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 29031 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 29032 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 29033 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 29033 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 29034 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 29035 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 29035 T16 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 29060 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 29064 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 29066 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 29069 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 29073 T16 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 29075 T16 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/ping,/update/json,/replication,spellCheckCompRH_Direct,/get,mock,/search-facet-invariants,mltrh,spellCheckWithWordbreak_Direct,/admin/plugins,/update/csv,/admin/file,/admin/mbeans,dismax,/search-facet-def,spellCheckWithWordbreak,defaults,/admin/segments,/admin/threads,/config,tvrh,/admin/properties,standard,/debug/dump,/update/json/docs,lazy,/mlt,spellCheckCompRH1,/schema,/terms,spellCheckCompRH,/admin/system,/admin/logging,/update,/admin/luke
   [junit4]   2> 29078 T16 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 29080 T16 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 29082 T16 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 29082 T16 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 29084 T16 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4b3fd89[collection1] main
   [junit4]   2> 29085 T16 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 29086 T16 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 29087 T16 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 29088 T16 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 29092 T16 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 29111 T16 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 29112 T23 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 29112 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
   [junit4]   2> 29113 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
   [junit4]   2> 29114 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Closing old IndexWriter... core=collection1
   [junit4]   2> 29114 T23 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 29114 T23 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 29115 T23 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 29115 T16 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 29116 T23 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 29118 T23 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 29119 T23 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 29120 T16 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7d97bf8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a07f17),segFN=segments_2,generation=2}
   [junit4]   2> 29120 T23 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 29120 T16 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 29121 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
   [junit4]   2> 29121 T23 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 29122 T16 c:collection1 oasc.SolrCore.getSearcher WARN [collection1] PERFORMANCE WARNING: Overlapping onDeckSearchers=2
   [junit4]   2> 29123 T23 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4b3fd89[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):C1)))}
   [junit4]   2> 29128 T16 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@7f3b346b[collection1] main
   [junit4]   2> 29130 T16 c:collection1 oasc.CoreContainer.registerCore replacing core: collection1
   [junit4]   2> 29130 T23 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7f3b346b[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):C1)))}
   [junit4]   2> 29130 T16 c:collection1 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@58169f1b
   [junit4]   2> 29132 T16 c:collection1 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=2,autocommit maxTime=1200ms,autocommits=1,soft autocommit maxTime=500ms,soft autocommits=1,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=1,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=143,transaction_logs_total_number=1}
   [junit4]   2> 29133 T16 c:collection1 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> ASYNC  NEW_CORE C1 name=collection1 org.apache.solr.core.SolrCore@1073c45f
   [junit4]   2> 29141 T16 c:collection1 C1 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[529 (1497736547077718016)]} 0 6
   [junit4]   2> 29144 T16 c:collection1 C1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 29340 T16 c:collection1 C1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7d97bf8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a07f17),segFN=segments_2,generation=2}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7d97bf8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a07f17),segFN=segments_3,generation=3}
   [junit4]   2> 29341 T16 c:collection1 C1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
   [junit4]   2> 29342 T16 c:collection1 C1 oass.SolrIndexSearcher.<init> Opening Searcher@61563c31[collection1] main
   [junit4]   2> 29343 T16 c:collection1 C1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 29344 T23 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@61563c31[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(5.2.0):C1)))}
   [junit4]   2> 29346 T16 c:collection1 C1 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 203
   [junit4]   2> 29356 T16 c:collection1 C1 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {delete=[529 (-1497736547307356160)]} 0 3
   [junit4]   2> 29858 T25 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 29868 T25 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4e22d17[collection1] main
   [junit4]   2> 29869 T25 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 29869 T23 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4e22d17[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 29873 T16 c:collection1 C1 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[550 (1497736547848421376)]} 0 3
   [junit4]   2> 30374 T25 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 30389 T25 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@71794a83[collection1] main
   [junit4]   2> 30389 T25 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 30390 T23 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@71794a83[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_2(5.2.0):C1)))}
   [junit4]   2> 30558 T24 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 30560 T24 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7d97bf8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a07f17),segFN=segments_3,generation=3}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7d97bf8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a07f17),segFN=segments_4,generation=4}
   [junit4]   2> 30560 T24 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 4
   [junit4]   2> 30561 T24 c:collection1 oasc.SolrCore.openNewSearcher SolrIndexSearcher has not changed - not re-opening: org.apache.solr.search.SolrIndexSearcher
   [junit4]   2> 30562 T24 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 32562 T16 c:collection1 oas.SolrTestCaseJ4.tearDown ###Ending testSoftAndHardCommitMaxTimeDelete
   [junit4]   2> 32571 T16 c:collection1 oas.SolrTestCaseJ4.setUp ###Starting testSoftAndHardCommitMaxTimeRapidAdds
   [junit4]   2> 32572 T16 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 32573 T16 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 32574 T16 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 32574 T16 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 32684 T16 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 32714 T16 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 32827 T16 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 32828 T16 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 32868 T16 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 33504 T16 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 33535 T16 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 33540 T16 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 33574 T16 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 33587 T16 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 33594 T16 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 33596 T16 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 33596 T16 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 33597 T16 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 33598 T16 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 33599 T16 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 33600 T16 c:collection1 oasc.CoreContainer.reload Reloading SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 33601 T16 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001/
   [junit4]   2> 33601 T16 c:collection1 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 33602 T16 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001/index/
   [junit4]   2> 33607 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 33608 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 33608 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 33609 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 33610 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 33611 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 33611 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 33612 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 33613 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 33614 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 33615 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 33617 T16 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 33640 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 33643 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 33646 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 33649 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 33654 T16 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 33657 T16 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/ping,/update/json,/replication,spellCheckCompRH_Direct,/get,mock,/search-facet-invariants,mltrh,spellCheckWithWordbreak_Direct,/admin/plugins,/update/csv,/admin/file,/admin/mbeans,dismax,/search-facet-def,spellCheckWithWordbreak,defaults,/admin/segments,/admin/threads,/config,tvrh,/admin/properties,standard,/debug/dump,/update/json/docs,lazy,/mlt,spellCheckCompRH1,/schema,/terms,spellCheckCompRH,/admin/system,/admin/logging,/update,/admin/luke
   [junit4]   2> 33660 T16 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 33662 T16 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 33663 T16 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 33664 T16 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 33666 T16 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@1879cd46[collection1] main
   [junit4]   2> 33667 T16 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 33668 T16 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 33668 T16 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 33669 T16 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 33694 T16 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 33709 T16 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 33711 T26 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 33711 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
   [junit4]   2> 33712 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
   [junit4]   2> 33712 T26 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 33713 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Closing old IndexWriter... core=collection1
   [junit4]   2> 33713 T26 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 33714 T26 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 33716 T26 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 33717 T26 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 33719 T26 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 33720 T16 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@28a0eaa9
   [junit4]   2> 33721 T26 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 33723 T26 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 33723 T26 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1879cd46[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_2(5.2.0):C1)))}
   [junit4]   2> 33727 T16 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7d97bf8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a07f17),segFN=segments_4,generation=4}
   [junit4]   2> 33728 T16 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 4
   [junit4]   2> 33729 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
   [junit4]   2> 33739 T16 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@7cc8ff30[collection1] main
   [junit4]   2> 33741 T16 c:collection1 oasc.CoreContainer.registerCore replacing core: collection1
   [junit4]   2> 33741 T26 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7cc8ff30[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_2(5.2.0):C1)))}
   [junit4]   2> 33742 T16 c:collection1 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@1073c45f
   [junit4]   2> 33745 T16 c:collection1 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=4,autocommit maxTime=1200ms,autocommits=1,soft autocommit maxTime=500ms,soft autocommits=2,optimizes=0,

[...truncated too long message...]

1 c:collection1 oasc.SocketProxy.close WARN Closing 2 connections to: http://127.0.0.1:63890/brdco/r
   [junit4]   2> 1504344 T2393 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:43349/brdco/r
   [junit4]   2> 1504344 T2393 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:41051/brdco/r
   [junit4]   2> 1504345 T2393 C:control_collection S:shard1 c:collection1 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1504345 T2393 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:43336/brdco/r
   [junit4]   2> 1504345 T2393 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:63890/brdco/r
   [junit4]   2> 1504345 T2393 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:43349/brdco/r
   [junit4]   2> 1504345 T2393 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:41051/brdco/r
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=E74DB75054A0944 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=nl_NL -Dtests.timezone=Antarctica/McMurdo -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   78.9s J0 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:43336 failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([E74DB75054A0944:8620E4AFABB664BC]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:43336 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1504360 T2393 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E74DB75054A0944-001
   [junit4]   2> 78855 T2392 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {range_facet_l=Lucene50(blocksize=128), multiDefault=Lucene50(blocksize=128), timestamp=Lucene50(blocksize=128), a_t=Lucene50(blocksize=128), intDefault=PostingsFormat(name=LuceneVarGapDocFreqInterval), _version_=PostingsFormat(name=LuceneVarGapDocFreqInterval), id=PostingsFormat(name=LuceneFixedGap), text=PostingsFormat(name=Memory doPackFST= true)}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=nl_NL, timezone=Antarctica/McMurdo
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=140790216,total=342360064
   [junit4]   2> NOTE: All tests run in this JVM: [EnumFieldTest, TestFaceting, DisMaxRequestHandlerTest, TestSolr4Spatial, TermVectorComponentTest, ExternalFileFieldSortTest, CircularListTest, OpenExchangeRatesOrgProviderTest, RecoveryAfterSoftCommitTest, MultiTermTest, LukeRequestHandlerTest, SaslZkACLProviderTest, JsonLoaderTest, StatelessScriptUpdateProcessorFactoryTest, HdfsUnloadDistributedZkTest, RequiredFieldsTest, TestPostingsSolrHighlighter, RemoteQueryErrorTest, DistributedDebugComponentTest, TestSchemaSimilarityResource, HdfsDirectoryTest, CoreMergeIndexesAdminHandlerTest, DocExpirationUpdateProcessorFactoryTest, TestLMDirichletSimilarityFactory, ShardRoutingCustomTest, TestUpdate, SuggestComponentTest, TestAddFieldRealTimeGet, TestSolrQueryParserDefaultOperatorResource, JSONWriterTest, MultiThreadedOCPTest, TestFuzzyAnalyzedSuggestions, TestClassNameShortening, UUIDUpdateProcessorFallbackTest, TestRebalanceLeaders, SpellCheckComponentTest, TestReload, TestExpandComponent, TestSchemaResource, TestExactSharedStatsCache, CSVRequestHandlerTest, TestAnalyzedSuggestions, TestHashPartitioner, IndexSchemaTest, BasicFunctionalityTest, LoggingHandlerTest, TestStressVersions, IndexBasedSpellCheckerTest, TestNoOpRegenerator, TestSolrJ, PreAnalyzedUpdateProcessorTest, SpellCheckCollatorTest, TestCodecSupport, TestBlobHandler, TestCloudManagedSchema, TestFieldResource, TestIntervalFaceting, BadIndexSchemaTest, LeaderFailoverAfterPartitionTest, TestCloudSchemaless, ReplicationFactorTest]
   [junit4] Completed on J0 in 79.38s, 1 test, 1 error <<< FAILURES!

[...truncated 727 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 479 suites, 1903 tests, 3 errors, 1 failure, 61 ignored (21 assumptions)

Total time: 53 minutes 54 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 223598798 bytes
Compression is 0.0%
Took 41 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



Re: [JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2900 - Still Failing

Posted by Mark Miller <ma...@gmail.com>.
Thats a bad fail. Something has probably changed in recovery code.

 - Mark

On Mon, Apr 6, 2015 at 2:11 PM Apache Jenkins Server <
jenkins@builds.apache.org> wrote:

> Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2900/
>
> 4 tests failed.
> REGRESSION:  org.apache.solr.cloud.RecoveryZkTest.test
>
> Error Message:
> shard1 is not consistent.  Got 1023 from http://127.0.0.1:10050/p_/z/
> collection1lastClient and got 295 from http://127.0.0.1:10059/p_/z/
> collection1
>
> Stack Trace:
> java.lang.AssertionError: shard1 is not consistent.  Got 1023 from
> http://127.0.0.1:10050/p_/z/collection1lastClient and got 295 from
> http://127.0.0.1:10059/p_/z/collection1
>         at __randomizedtesting.SeedInfo.seed([FBFBECE5D5AABA29:
> 73AFD33F7B56D7D1]:0)
>         at org.junit.Assert.fail(Assert.java:93)
>         at org.apache.solr.cloud.RecoveryZkTest.test(
> RecoveryZkTest.java:123)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(
> NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(
> DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(
> RandomizedRunner.java:1627)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(
> RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(
> RandomizedRunner.java:872)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(
> RandomizedRunner.java:886)
>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$
> ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$
> ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
>         at com.carrotsearch.randomizedtesting.rules.
> SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.
> java:53)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(
> TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(
> AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(
> TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures
> $1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.
> evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$
> StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.
> forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.
> ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.
> runSingleTest(RandomizedRunner.java:845)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(
> RandomizedRunner.java:747)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(
> RandomizedRunner.java:781)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(
> RandomizedRunner.java:792)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.
> java:53)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(
> AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(
> TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.
> NoShadowingOrOverridesOnMethodsRule$1.evaluate(
> NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.
> NoShadowingOrOverridesOnMethodsRule$1.evaluate(
> NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(
> TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.
> evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures
> $1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(
> TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$
> StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
>
>
> FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test
>
> Error Message:
> IOException occured when talking to server at:
> http://127.0.0.1:52540/fmq/gz/c8n_1x3_commits_shard1_replica1
>
> Stack Trace:
> org.apache.solr.client.solrj.SolrServerException: IOException occured
> when talking to server at: http://127.0.0.1:52540/fmq/gz/
> c8n_1x3_commits_shard1_replica1
>         at __randomizedtesting.SeedInfo.seed([FBFBECE5D5AABA29:
> 73AFD33F7B56D7D1]:0)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.
> executeMethod(HttpSolrClient.java:570)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(
> HttpSolrClient.java:233)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(
> HttpSolrClient.java:225)
>         at org.apache.solr.client.solrj.SolrRequest.process(
> SolrRequest.java:135)
>         at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.
> java:483)
>         at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.
> java:464)
>         at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommi
> tTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
>         at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(
> LeaderInitiatedRecoveryOnCommitTest.java:64)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(
> NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(
> DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(
> RandomizedRunner.java:1627)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(
> RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(
> RandomizedRunner.java:872)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(
> RandomizedRunner.java:886)
>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$
> ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$
> ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
>         at com.carrotsearch.randomizedtesting.rules.
> SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.
> java:53)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(
> TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(
> AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(
> TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures
> $1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.
> evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$
> StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.
> forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.
> ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.
> runSingleTest(RandomizedRunner.java:845)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(
> RandomizedRunner.java:747)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(
> RandomizedRunner.java:781)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(
> RandomizedRunner.java:792)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.
> java:53)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(
> AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(
> TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.
> NoShadowingOrOverridesOnMethodsRule$1.evaluate(
> NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.
> NoShadowingOrOverridesOnMethodsRule$1.evaluate(
> NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(
> TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.
> evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures
> $1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(
> TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$
> StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
> Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:52540
> failed to respond
>         at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(
> DefaultHttpResponseParser.java:143)
>         at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(
> DefaultHttpResponseParser.java:57)
>         at org.apache.http.impl.io.AbstractMessageParser.parse(
> AbstractMessageParser.java:261)
>         at org.apache.http.impl.AbstractHttpClientConnection.
> receiveResponseHeader(AbstractHttpClientConnection.java:283)
>         at org.apache.http.impl.conn.DefaultClientConnection.
> receiveResponseHeader(DefaultClientConnection.java:251)
>         at org.apache.http.impl.conn.ManagedClientConnectionImpl.
> receiveResponseHeader(ManagedClientConnectionImpl.java:197)
>         at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(
> HttpRequestExecutor.java:272)
>         at org.apache.http.protocol.HttpRequestExecutor.execute(
> HttpRequestExecutor.java:124)
>         at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(
> DefaultRequestDirector.java:685)
>         at org.apache.http.impl.client.DefaultRequestDirector.execute(
> DefaultRequestDirector.java:487)
>         at org.apache.http.impl.client.AbstractHttpClient.doExecute(
> AbstractHttpClient.java:882)
>         at org.apache.http.impl.client.CloseableHttpClient.execute(
> CloseableHttpClient.java:82)
>         at org.apache.http.impl.client.CloseableHttpClient.execute(
> CloseableHttpClient.java:107)
>         at org.apache.http.impl.client.CloseableHttpClient.execute(
> CloseableHttpClient.java:55)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.
> executeMethod(HttpSolrClient.java:462)
>         ... 48 more
>
>
> FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test
>
> Error Message:
> org.apache.solr.client.solrj.SolrServerException: IOException occured
> when talking to server at: http://127.0.0.1:50643/collection1
>
> Stack Trace:
> org.apache.solr.client.solrj.SolrServerException:
> org.apache.solr.client.solrj.SolrServerException: IOException occured
> when talking to server at: http://127.0.0.1:50643/collection1
>         at __randomizedtesting.SeedInfo.seed([FBFBECE5D5AABA29:
> 73AFD33F7B56D7D1]:0)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.
> directUpdate(CloudSolrClient.java:625)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.
> sendRequest(CloudSolrClient.java:948)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.
> requestWithRetryOnStaleState(CloudSolrClient.java:839)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.request(
> CloudSolrClient.java:782)
>         at org.apache.solr.client.solrj.SolrRequest.process(
> SolrRequest.java:135)
>         at org.apache.solr.client.solrj.SolrClient.add(SolrClient.
> java:174)
>         at org.apache.solr.client.solrj.SolrClient.add(SolrClient.
> java:139)
>         at org.apache.solr.client.solrj.SolrClient.add(SolrClient.
> java:153)
>         at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(
> RecoveryAfterSoftCommitTest.java:88)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(
> NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(
> DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(
> RandomizedRunner.java:1627)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(
> RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(
> RandomizedRunner.java:872)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(
> RandomizedRunner.java:886)
>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$
> ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$
> ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
>         at com.carrotsearch.randomizedtesting.rules.
> SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.
> java:53)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(
> TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(
> AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(
> TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures
> $1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.
> evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$
> StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.
> forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.
> ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.
> runSingleTest(RandomizedRunner.java:845)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(
> RandomizedRunner.java:747)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(
> RandomizedRunner.java:781)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(
> RandomizedRunner.java:792)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.
> java:53)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(
> AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(
> TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.
> NoShadowingOrOverridesOnMethodsRule$1.evaluate(
> NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.
> NoShadowingOrOverridesOnMethodsRule$1.evaluate(
> NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(
> TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.
> evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures
> $1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(
> TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$
> StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
> Caused by: org.apache.solr.client.solrj.SolrServerException: IOException
> occured when talking to server at: http://127.0.0.1:50643/collection1
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.
> executeMethod(HttpSolrClient.java:570)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(
> HttpSolrClient.java:233)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(
> HttpSolrClient.java:225)
>         at org.apache.solr.client.solrj.impl.LBHttpSolrClient.
> doRequest(LBHttpSolrClient.java:370)
>         at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(
> LBHttpSolrClient.java:325)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.
> directUpdate(CloudSolrClient.java:622)
>         ... 49 more
> Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:50643
> failed to respond
>         at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(
> DefaultHttpResponseParser.java:143)
>         at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(
> DefaultHttpResponseParser.java:57)
>         at org.apache.http.impl.io.AbstractMessageParser.parse(
> AbstractMessageParser.java:261)
>         at org.apache.http.impl.AbstractHttpClientConnection.
> receiveResponseHeader(AbstractHttpClientConnection.java:283)
>         at org.apache.http.impl.conn.DefaultClientConnection.
> receiveResponseHeader(DefaultClientConnection.java:251)
>         at org.apache.http.impl.conn.ManagedClientConnectionImpl.
> receiveResponseHeader(ManagedClientConnectionImpl.java:197)
>         at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(
> HttpRequestExecutor.java:272)
>         at org.apache.http.protocol.HttpRequestExecutor.execute(
> HttpRequestExecutor.java:124)
>         at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(
> DefaultRequestDirector.java:685)
>         at org.apache.http.impl.client.DefaultRequestDirector.execute(
> DefaultRequestDirector.java:487)
>         at org.apache.http.impl.client.AbstractHttpClient.doExecute(
> AbstractHttpClient.java:882)
>         at org.apache.http.impl.client.CloseableHttpClient.execute(
> CloseableHttpClient.java:82)
>         at org.apache.http.impl.client.CloseableHttpClient.execute(
> CloseableHttpClient.java:107)
>         at org.apache.http.impl.client.CloseableHttpClient.execute(
> CloseableHttpClient.java:55)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.
> executeMethod(HttpSolrClient.java:462)
>         ... 54 more
>
>
> FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test
>
> Error Message:
> org.apache.solr.client.solrj.SolrServerException: IOException occured
> when talking to server at: http://127.0.0.1:11223/
> repfacttest_c8n_1x3_shard1_replica2
>
> Stack Trace:
> org.apache.solr.client.solrj.SolrServerException:
> org.apache.solr.client.solrj.SolrServerException: IOException occured
> when talking to server at: http://127.0.0.1:11223/
> repfacttest_c8n_1x3_shard1_replica2
>         at __randomizedtesting.SeedInfo.seed([FBFBECE5D5AABA29:
> 73AFD33F7B56D7D1]:0)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.
> directUpdate(CloudSolrClient.java:625)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.
> sendRequest(CloudSolrClient.java:948)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.
> requestWithRetryOnStaleState(CloudSolrClient.java:839)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.request(
> CloudSolrClient.java:782)
>         at org.apache.solr.client.solrj.SolrClient.request(SolrClient.
> java:1220)
>         at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(
> ReplicationFactorTest.java:308)
>         at org.apache.solr.cloud.ReplicationFactorTest.testRf3(
> ReplicationFactorTest.java:234)
>         at org.apache.solr.cloud.ReplicationFactorTest.test(
> ReplicationFactorTest.java:113)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(
> NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(
> DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(
> RandomizedRunner.java:1627)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(
> RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(
> RandomizedRunner.java:872)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(
> RandomizedRunner.java:886)
>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$
> ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$
> ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
>         at com.carrotsearch.randomizedtesting.rules.
> SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.
> java:53)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(
> TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(
> AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(
> TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures
> $1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.
> evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$
> StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.
> forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.
> ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.
> runSingleTest(RandomizedRunner.java:845)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(
> RandomizedRunner.java:747)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(
> RandomizedRunner.java:781)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(
> RandomizedRunner.java:792)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.
> java:53)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(
> AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(
> TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.
> NoShadowingOrOverridesOnMethodsRule$1.evaluate(
> NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.
> NoShadowingOrOverridesOnMethodsRule$1.evaluate(
> NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(
> TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.
> evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures
> $1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(
> TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$
> StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
> Caused by: org.apache.solr.client.solrj.SolrServerException: IOException
> occured when talking to server at: http://127.0.0.1:11223/
> repfacttest_c8n_1x3_shard1_replica2
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.
> executeMethod(HttpSolrClient.java:570)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(
> HttpSolrClient.java:233)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(
> HttpSolrClient.java:225)
>         at org.apache.solr.client.solrj.impl.LBHttpSolrClient.
> doRequest(LBHttpSolrClient.java:370)
>         at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(
> LBHttpSolrClient.java:325)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.
> directUpdate(CloudSolrClient.java:622)
>         ... 48 more
> Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:11223
> failed to respond
>         at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(
> DefaultHttpResponseParser.java:143)
>         at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(
> DefaultHttpResponseParser.java:57)
>         at org.apache.http.impl.io.AbstractMessageParser.parse(
> AbstractMessageParser.java:261)
>         at org.apache.http.impl.AbstractHttpClientConnection.
> receiveResponseHeader(AbstractHttpClientConnection.java:283)
>         at org.apache.http.impl.conn.DefaultClientConnection.
> receiveResponseHeader(DefaultClientConnection.java:251)
>         at org.apache.http.impl.conn.ManagedClientConnectionImpl.
> receiveResponseHeader(ManagedClientConnectionImpl.java:197)
>         at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(
> HttpRequestExecutor.java:272)
>         at org.apache.http.protocol.HttpRequestExecutor.execute(
> HttpRequestExecutor.java:124)
>         at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(
> DefaultRequestDirector.java:685)
>         at org.apache.http.impl.client.DefaultRequestDirector.execute(
> DefaultRequestDirector.java:487)
>         at org.apache.http.impl.client.AbstractHttpClient.doExecute(
> AbstractHttpClient.java:882)
>         at org.apache.http.impl.client.CloseableHttpClient.execute(
> CloseableHttpClient.java:82)
>         at org.apache.http.impl.client.CloseableHttpClient.execute(
> CloseableHttpClient.java:107)
>         at org.apache.http.impl.client.CloseableHttpClient.execute(
> CloseableHttpClient.java:55)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.
> executeMethod(HttpSolrClient.java:462)
>         ... 53 more
>
>
>
>
> Build Log:
> [...truncated 9494 lines...]
>    [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
>    [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/init-core-data-001
>    [junit4]   2> 295434 T1269 oas.BaseDistributedSearchTestCase.initHostContext
> Setting hostContext system property: /
>    [junit4]   2> 295441 T1269 oasc.ZkTestServer.run STARTING ZK TEST SERVER
>    [junit4]   1> client port:0.0.0.0/0.0.0.0:0
>    [junit4]   2> 295441 T1270 oasc.ZkTestServer$ZKServerMain.runFromConfig
> Starting server
>    [junit4]   2> 295541 T1269 oasc.ZkTestServer.run start zk server on
> port:50623
>    [junit4]   2> 295542 T1269 oascc.SolrZkClient.
> createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
>    [junit4]   2> 295542 T1269 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>    [junit4]   2> 295545 T1277 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@6e5605f4
> name:ZooKeeperConnection Watcher:127.0.0.1:50623 got event WatchedEvent
> state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 295545 T1269 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>    [junit4]   2> 295546 T1269 oascc.SolrZkClient.createZkACLProvider
> Using default ZkACLProvider
>    [junit4]   2> 295546 T1269 oascc.SolrZkClient.makePath makePath: /solr
>    [junit4]   2> 295548 T1269 oascc.SolrZkClient.
> createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
>    [junit4]   2> 295549 T1269 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>    [junit4]   2> 295550 T1280 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@842caa6
> name:ZooKeeperConnection Watcher:127.0.0.1:50623/solr got event
> WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 295550 T1269 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>    [junit4]   2> 295551 T1269 oascc.SolrZkClient.createZkACLProvider
> Using default ZkACLProvider
>    [junit4]   2> 295551 T1269 oascc.SolrZkClient.makePath makePath:
> /collections/collection1
>    [junit4]   2> 295553 T1269 oascc.SolrZkClient.makePath makePath:
> /collections/collection1/shards
>    [junit4]   2> 295554 T1269 oascc.SolrZkClient.makePath makePath:
> /collections/control_collection
>    [junit4]   2> 295555 T1269 oascc.SolrZkClient.makePath makePath:
> /collections/control_collection/shards
>    [junit4]   2> 295556 T1269 oasc.AbstractZkTestCase.putConfig put
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
> to /configs/conf1/solrconfig.xml
>    [junit4]   2> 295557 T1269 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/solrconfig.xml
>    [junit4]   2> 295559 T1269 oasc.AbstractZkTestCase.putConfig put
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
> to /configs/conf1/schema.xml
>    [junit4]   2> 295560 T1269 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/schema.xml
>    [junit4]   2> 295661 T1269 oasc.AbstractZkTestCase.putConfig put
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
> conf/solrconfig.snippet.randomindexconfig.xml to
> /configs/conf1/solrconfig.snippet.randomindexconfig.xml
>    [junit4]   2> 295662 T1269 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/solrconfig.snippet.randomindexconfig.xml
>    [junit4]   2> 295663 T1269 oasc.AbstractZkTestCase.putConfig put
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt
> to /configs/conf1/stopwords.txt
>    [junit4]   2> 295663 T1269 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/stopwords.txt
>    [junit4]   2> 295665 T1269 oasc.AbstractZkTestCase.putConfig put
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt
> to /configs/conf1/protwords.txt
>    [junit4]   2> 295665 T1269 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/protwords.txt
>    [junit4]   2> 295667 T1269 oasc.AbstractZkTestCase.putConfig put
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml
> to /configs/conf1/currency.xml
>    [junit4]   2> 295667 T1269 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/currency.xml
>    [junit4]   2> 295668 T1269 oasc.AbstractZkTestCase.putConfig put
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml
> to /configs/conf1/enumsConfig.xml
>    [junit4]   2> 295669 T1269 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/enumsConfig.xml
>    [junit4]   2> 295670 T1269 oasc.AbstractZkTestCase.putConfig put
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json
> to /configs/conf1/open-exchange-rates.json
>    [junit4]   2> 295670 T1269 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/open-exchange-rates.json
>    [junit4]   2> 295672 T1269 oasc.AbstractZkTestCase.putConfig put
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
> conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-
> ISOLatin1Accent.txt
>    [junit4]   2> 295672 T1269 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/mapping-ISOLatin1Accent.txt
>    [junit4]   2> 295673 T1269 oasc.AbstractZkTestCase.putConfig put
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt
> to /configs/conf1/old_synonyms.txt
>    [junit4]   2> 295674 T1269 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/old_synonyms.txt
>    [junit4]   2> 295675 T1269 oasc.AbstractZkTestCase.putConfig put
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt
> to /configs/conf1/synonyms.txt
>    [junit4]   2> 295675 T1269 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/synonyms.txt
>    [junit4]   2> 296963 T1269 oas.SolrTestCaseJ4.writeCoreProperties
> Writing core.properties file to /usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores/collection1
>    [junit4]   2> 296967 T1269 oejs.Server.doStart jetty-8.1.10.v20130312
>    [junit4]   2> 296971 T1269 oejs.AbstractConnector.doStart Started
> SocketConnector@127.0.0.1:50636
>    [junit4]   2> 296971 T1269 oascse.JettySolrRunner$1.lifeCycleStarted
> Jetty properties: {hostPort=50635, coreRootDirectory=/usr/home/
> jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.
> x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores, hostContext=/,
> solr.data.dir=/usr/home/jenkins/jenkins-slave/
> workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/
> test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/tempDir-001/control/data}
>    [junit4]   2> 296971 T1269 oass.SolrDispatchFilter.init
> SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
>    [junit4]   2> 296972 T1269 oasc.SolrResourceLoader.<init> new
> SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/'
>    [junit4]   2> 297015 T1269 oasc.SolrXmlConfig.fromFile Loading
> container configuration from /usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/solr.xml
>    [junit4]   2> 297067 T1269 oasc.CorePropertiesLocator.<init>
> Config-defined core root directory: /usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores
>    [junit4]   2> 297069 T1269 oasc.CoreContainer.<init> New CoreContainer
> 2110879613
>    [junit4]   2> 297069 T1269 oasc.CoreContainer.load Loading cores into
> CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/
> workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/
> test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/]
>    [junit4]   2> 297069 T1269 oasc.CoreContainer.load loading shared
> library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/lib
>    [junit4]   2> 297070 T1269 oasc.SolrResourceLoader.addToClassLoader
> WARN Can't find (or read) directory to add to classloader: lib (resolved
> as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/lib).
>    [junit4]   2> 297080 T1269 oashc.HttpShardHandlerFactory.getParameter
> Setting socketTimeout to: 90000
>    [junit4]   2> 297080 T1269 oashc.HttpShardHandlerFactory.getParameter
> Setting urlScheme to:
>    [junit4]   2> 297080 T1269 oashc.HttpShardHandlerFactory.getParameter
> Setting connTimeout to: 15000
>    [junit4]   2> 297080 T1269 oashc.HttpShardHandlerFactory.getParameter
> Setting maxConnectionsPerHost to: 20
>    [junit4]   2> 297081 T1269 oashc.HttpShardHandlerFactory.getParameter
> Setting maxConnections to: 10000
>    [junit4]   2> 297081 T1269 oashc.HttpShardHandlerFactory.getParameter
> Setting corePoolSize to: 0
>    [junit4]   2> 297081 T1269 oashc.HttpShardHandlerFactory.getParameter
> Setting maximumPoolSize to: 2147483647
>    [junit4]   2> 297081 T1269 oashc.HttpShardHandlerFactory.getParameter
> Setting maxThreadIdleTime to: 5
>    [junit4]   2> 297082 T1269 oashc.HttpShardHandlerFactory.getParameter
> Setting sizeOfQueue to: -1
>    [junit4]   2> 297082 T1269 oashc.HttpShardHandlerFactory.getParameter
> Setting fairnessPolicy to: false
>    [junit4]   2> 297082 T1269 oashc.HttpShardHandlerFactory.getParameter
> Setting useRetries to: false
>    [junit4]   2> 297089 T1269 oasu.UpdateShardHandler.<init> Creating
> UpdateShardHandler HTTP client with params: socketTimeout=340000&
> connTimeout=45000&retry=true
>    [junit4]   2> 297090 T1269 oasl.LogWatcher.createWatcher SLF4J impl is
> org.slf4j.impl.Log4jLoggerFactory
>    [junit4]   2> 297090 T1269 oasl.LogWatcher.newRegisteredLogWatcher
> Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>    [junit4]   2> 297090 T1269 oasc.CoreContainer.load Node Name: 127.0.0.1
>    [junit4]   2> 297091 T1269 oasc.ZkContainer.initZooKeeper Zookeeper
> client=127.0.0.1:50623/solr
>    [junit4]   2> 297091 T1269 oasc.ZkController.checkChrootPath zkHost
> includes chroot
>    [junit4]   2> 297092 T1269 oascc.SolrZkClient.
> createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
>    [junit4]   2> 297092 T1269 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>    [junit4]   2> 297094 T1292 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@45cc06e8
> name:ZooKeeperConnection Watcher:127.0.0.1:50623 got event WatchedEvent
> state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 297095 T1269 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>    [junit4]   2> 297096 T1269 oascc.SolrZkClient.createZkACLProvider
> Using default ZkACLProvider
>    [junit4]   2> 297097 T1269 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>    [junit4]   2> 297098 T1295 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@34504d13
> name:ZooKeeperConnection Watcher:127.0.0.1:50623/solr got event
> WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 297098 T1269 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>    [junit4]   2> 297100 T1269 oascc.SolrZkClient.makePath makePath:
> /overseer/queue
>    [junit4]   2> 297101 T1269 oascc.SolrZkClient.makePath makePath:
> /overseer/collection-queue-work
>    [junit4]   2> 297103 T1269 oascc.SolrZkClient.makePath makePath:
> /overseer/collection-map-running
>    [junit4]   2> 297104 T1269 oascc.SolrZkClient.makePath makePath:
> /overseer/collection-map-completed
>    [junit4]   2> 297105 T1269 oascc.SolrZkClient.makePath makePath:
> /overseer/collection-map-failure
>    [junit4]   2> 297107 T1269 oascc.SolrZkClient.makePath makePath:
> /live_nodes
>    [junit4]   2> 297108 T1269 oascc.SolrZkClient.makePath makePath:
> /aliases.json
>    [junit4]   2> 297109 T1269 oascc.SolrZkClient.makePath makePath:
> /clusterstate.json
>    [junit4]   2> 297110 T1269 oasc.ZkController.createEphemeralLiveNode
> Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50635_
>    [junit4]   2> 297110 T1269 oascc.SolrZkClient.makePath makePath:
> /live_nodes/127.0.0.1:50635_
>    [junit4]   2> 297112 T1269 oascc.SolrZkClient.makePath makePath:
> /overseer_elect
>    [junit4]   2> 297113 T1269 oascc.SolrZkClient.makePath makePath:
> /overseer_elect/election
>    [junit4]   2> 297114 T1269 oasc.Overseer.close Overseer (id=null)
> closing
>    [junit4]   2> 297115 T1269 oasc.OverseerElectionContext.runLeaderProcess
> I am going to be the leader 127.0.0.1:50635_
>    [junit4]   2> 297115 T1269 oascc.SolrZkClient.makePath makePath:
> /overseer_elect/leader
>    [junit4]   2> 297116 T1269 oasc.Overseer.start Overseer
> (id=93607700461191171-127.0.0.1:50635_-n_0000000000) starting
>    [junit4]   2> 297118 T1269 oascc.SolrZkClient.makePath makePath:
> /overseer/queue-work
>    [junit4]   2> 297122 T1269 oasc.OverseerAutoReplicaFailoverThread.<init>
> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000
> autoReplicaFailoverWaitAfterExpiration=30000
> autoReplicaFailoverBadNodeExpiration=60000
>    [junit4]   2> 297122 T1297 oasc.OverseerCollectionProcessor.run
> Process current queue of collection creations
>    [junit4]   2> 297122 T1269 oascc.ZkStateReader.
> createClusterStateWatchersAndUpdate Updating cluster state from
> ZooKeeper...
>    [junit4]   2> 297122 T1296 oasc.Overseer$ClusterStateUpdater.run
> Starting to work on the main queue
>    [junit4]   2> 297126 T1269 oasc.CorePropertiesLocator.discover Looking
> for core definitions underneath /usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores
>    [junit4]   2> 297128 T1269 oasc.CoreDescriptor.<init> CORE DESCRIPTOR:
> {shard=, collection=control_collection, coreNodeName=, dataDir=data/,
> absoluteInstDir=/usr/home/jenkins/jenkins-slave/
> workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/
> test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores/collection1/,
> config=solrconfig.xml, schema=schema.xml, loadOnStartup=true,
> transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/
> Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores/collection1, name=collection1}
>    [junit4]   2> 297128 T1269 oasc.CorePropertiesLocator.discoverUnder
> Found core collection1 in /usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores/collection1/
>    [junit4]   2> 297128 T1269 oasc.CorePropertiesLocator.discover Found 1
> core definitions
>    [junit4]   2> 297130 T1299 C:control_collection c:collection1
> oasc.ZkController.publish publishing core=collection1 state=down
> collection=control_collection
>    [junit4]   2> 297130 T1299 C:control_collection c:collection1
> oasc.ZkController.publish numShards not found on descriptor - reading it
> from system property
>    [junit4]   2> 297131 T1299 C:control_collection c:collection1
> oasc.ZkController.waitForCoreNodeName look for our core node name
>    [junit4]   2> 297131 T1295 oasc.DistributedQueue$LatchWatcher.process
> NodeChildrenChanged fired on path /overseer/queue state SyncConnected
>    [junit4]   2> 297132 T1296 oasc.Overseer$ClusterStateUpdater.run
> processMessage: queueSize: 1, message = {
>    [junit4]   2>          "collection":"control_collection",
>    [junit4]   2>          "state":"down",
>    [junit4]   2>          "shard":null,
>    [junit4]   2>          "base_url":"http://127.0.0.1:50635",
>    [junit4]   2>          "roles":null,
>    [junit4]   2>          "node_name":"127.0.0.1:50635_",
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "operation":"state",
>    [junit4]   2>          "numShards":"1"} current state version: 0
>    [junit4]   2> 297132 T1296 oasco.ReplicaMutator.updateState Update
> state numShards=1 message={
>    [junit4]   2>          "collection":"control_collection",
>    [junit4]   2>          "state":"down",
>    [junit4]   2>          "shard":null,
>    [junit4]   2>          "base_url":"http://127.0.0.1:50635",
>    [junit4]   2>          "roles":null,
>    [junit4]   2>          "node_name":"127.0.0.1:50635_",
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "operation":"state",
>    [junit4]   2>          "numShards":"1"}
>    [junit4]   2> 297132 T1296 oasco.ClusterStateMutator.createCollection
> building a new cName: control_collection
>    [junit4]   2> 297133 T1296 oasco.ReplicaMutator.updateState Assigning
> new node to shard shard=shard1
>    [junit4]   2> 297134 T1295 oascc.ZkStateReader$2.process A cluster
> state change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 298131 T1299 C:control_collection c:collection1
> oasc.ZkController.waitForShardId waiting to find shard id in clusterstate
> for collection1
>    [junit4]   2> 298132 T1299 C:control_collection c:collection1
> oasc.ZkController.createCollectionZkNode Check for collection
> zkNode:control_collection
>    [junit4]   2> 298132 T1299 C:control_collection c:collection1
> oasc.ZkController.createCollectionZkNode Collection zkNode exists
>    [junit4]   2> 298132 T1299 C:control_collection c:collection1
> oascc.ZkStateReader.readConfigName Load collection config
> from:/collections/control_collection
>    [junit4]   2> 298133 T1299 C:control_collection c:collection1
> oascc.ZkStateReader.readConfigName path=/collections/control_collection
> configName=conf1 specified config exists in ZooKeeper
>    [junit4]   2> 298133 T1299 C:control_collection c:collection1
> oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory:
> '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores/collection1/'
>    [junit4]   2> 298147 T1299 C:control_collection c:collection1
> oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
>    [junit4]   2> 298149 T1299 C:control_collection c:collection1
> oasc.Config.<init> loaded config solrconfig.xml with version 0
>    [junit4]   2> 298156 T1299 C:control_collection c:collection1
> oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
>    [junit4]   2> 298172 T1299 C:control_collection c:collection1
> oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
>    [junit4]   2> 298210 T1299 C:control_collection c:collection1
> oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
>    [junit4]   2> 298311 T1299 C:control_collection c:collection1
> oass.IndexSchema.readSchema Reading Solr Schema from
> /configs/conf1/schema.xml
>    [junit4]   2> 298318 T1299 C:control_collection c:collection1
> oass.IndexSchema.readSchema [collection1] Schema name=test
>    [junit4]   2> 298623 T1299 C:control_collection c:collection1 oass.
> OpenExchangeRatesOrgProvider.init Initialized with
> rates=open-exchange-rates.json, refreshInterval=1440.
>    [junit4]   2> 298636 T1299 C:control_collection c:collection1
> oass.IndexSchema.readSchema default search field in schema is text
>    [junit4]   2> 298638 T1299 C:control_collection c:collection1
> oass.IndexSchema.readSchema unique key field: id
>    [junit4]   2> 298648 T1299 C:control_collection c:collection1
> oass.FileExchangeRateProvider.reload Reloading exchange rates from file
> currency.xml
>    [junit4]   2> 298651 T1299 C:control_collection c:collection1
> oass.FileExchangeRateProvider.reload Reloading exchange rates from file
> currency.xml
>    [junit4]   2> 298654 T1299 C:control_collection c:collection1 oass.
> OpenExchangeRatesOrgProvider.reload Reloading exchange rates from
> open-exchange-rates.json
>    [junit4]   2> 298655 T1299 C:control_collection c:collection1 oass.
> OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key
> IMPORTANT NOTE
>    [junit4]   2> 298655 T1299 C:control_collection c:collection1 oass.
> OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key,
> got STRING
>    [junit4]   2> 298655 T1299 C:control_collection c:collection1 oass.
> OpenExchangeRatesOrgProvider.reload Reloading exchange rates from
> open-exchange-rates.json
>    [junit4]   2> 298656 T1299 C:control_collection c:collection1 oass.
> OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key
> IMPORTANT NOTE
>    [junit4]   2> 298656 T1299 C:control_collection c:collection1 oass.
> OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key,
> got STRING
>    [junit4]   2> 298656 T1299 C:control_collection c:collection1
> oasc.CoreContainer.create Creating SolrCore 'collection1' using
> configuration from collection control_collection
>    [junit4]   2> 298656 T1299 C:control_collection c:collection1
> oasc.SolrCore.initDirectoryFactory org.apache.solr.core.
> MockDirectoryFactory
>    [junit4]   2> 298657 T1299 C:control_collection c:collection1
> oasc.SolrCore.<init> [collection1] Opening new SolrCore at
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores/collection1/,
> dataDir=/usr/home/jenkins/jenkins-slave/workspace/
> Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores/collection1/data/
>    [junit4]   2> 298657 T1299 C:control_collection c:collection1
> oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans
> to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ec2b6ef
>    [junit4]   2> 298658 T1299 C:control_collection c:collection1
> oasc.CachingDirectoryFactory.get return new directory for
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores/collection1/data
>    [junit4]   2> 298658 T1299 C:control_collection c:collection1
> oasc.SolrCore.getNewIndexDir New index directory detected: old=null
> new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores/collection1/data/index/
>    [junit4]   2> 298658 T1299 C:control_collection c:collection1
> oasc.SolrCore.initIndex WARN [collection1] Solr index directory
> '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores/collection1/data/index' doesn't
> exist. Creating new index...
>    [junit4]   2> 298658 T1299 C:control_collection c:collection1
> oasc.CachingDirectoryFactory.get return new directory for
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores/collection1/data/index
>    [junit4]   2> 298659 T1299 C:control_collection c:collection1
> oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class
> org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy:
> maxMergeAtOnce=27, maxMergeAtOnceExplicit=50, maxMergedSegmentMB=52.0615234375,
> floorSegmentMB=0.486328125, forceMergeDeletesPctAllowed=10.112383767408401,
> segmentsPerTier=19.0, maxCFSSegmentSizeMB=8.796093022207999E12,
> noCFSRatio=1.0
>    [junit4]   2> 298659 T1299 C:control_collection c:collection1
> oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits:
> num=1
>    [junit4]   2>                commit{dir=MockDirectoryWrapper(
> RAMDirectory@6041646a lockFactory=org.apache.lucene.store.
> SingleInstanceLockFactory@65279c84),segFN=segments_1,generation=1}
>    [junit4]   2> 298659 T1299 C:control_collection c:collection1
> oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>    [junit4]   2> 298664 T1299 C:control_collection c:collection1 oasup.
> UpdateRequestProcessorChain.init creating updateRequestProcessorChain
> "nodistrib"
>    [junit4]   2> 298664 T1299 C:control_collection c:collection1 oasup.
> UpdateRequestProcessorChain.init creating updateRequestProcessorChain
> "dedupe"
>    [junit4]   2> 298664 T1299 C:control_collection c:collection1 oasup.
> UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory
> into updateRequestProcessorChain "dedupe"
>    [junit4]   2> 298664 T1299 C:control_collection c:collection1 oasup.
> UpdateRequestProcessorChain.init creating updateRequestProcessorChain
> "stored_sig"
>    [junit4]   2> 298665 T1299 C:control_collection c:collection1 oasup.
> UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory
> into updateRequestProcessorChain "stored_sig"
>    [junit4]   2> 298665 T1299 C:control_collection c:collection1 oasup.
> UpdateRequestProcessorChain.init creating updateRequestProcessorChain
> "distrib-dup-test-chain-explicit"
>    [junit4]   2> 298665 T1299 C:control_collection c:collection1 oasup.
> UpdateRequestProcessorChain.init creating updateRequestProcessorChain
> "distrib-dup-test-chain-implicit"
>    [junit4]   2> 298665 T1299 C:control_collection c:collection1 oasup.
> UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory
> into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>    [junit4]   2> 298666 T1299 C:control_collection c:collection1
> oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain
> defined as default, creating implicit default
>    [junit4]   2> 298685 T1299 C:control_collection c:collection1
> oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>    [junit4]   2> 298687 T1299 C:control_collection c:collection1
> oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>    [junit4]   2> 298689 T1299 C:control_collection c:collection1
> oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>    [junit4]   2> 298690 T1299 C:control_collection c:collection1
> oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>    [junit4]   2> 298694 T1299 C:control_collection c:collection1
> oasc.RequestHandlers.initHandlersFromConfig Registered paths:
> /schema,/update/json/docs,/admin/threads,/config,/
> replication,/admin/file,/admin/logging,/admin/plugins,/
> update/csv,standard,/admin/ping,/admin/system,/get,/
> admin/luke,/admin/segments,/admin/mbeans,/admin/
> properties,/update,/update/json
>    [junit4]   2> 298696 T1299 C:control_collection c:collection1
> oasc.SolrCore.initStatsCache Using default statsCache cache:
> org.apache.solr.search.stats.LocalStatsCache
>    [junit4]   2> 298697 T1299 C:control_collection c:collection1
> oasu.UpdateHandler.<init> Using UpdateLog implementation:
> org.apache.solr.update.UpdateLog
>    [junit4]   2> 298697 T1299 C:control_collection c:collection1
> oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH
> numRecordsToKeep=100 maxNumLogsToKeep=10
>    [junit4]   2> 298699 T1299 C:control_collection c:collection1
> oasu.CommitTracker.<init> Hard AutoCommit: disabled
>    [junit4]   2> 298699 T1299 C:control_collection c:collection1
> oasu.CommitTracker.<init> Soft AutoCommit: disabled
>    [junit4]   2> 298699 T1299 C:control_collection c:collection1
> oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class
> org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy:
> maxMergeAtOnce=30, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=39.86328125,
> floorSegmentMB=2.154296875, forceMergeDeletesPctAllowed=25.115751995354497,
> segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12,
> noCFSRatio=0.0
>    [junit4]   2> 298700 T1299 C:control_collection c:collection1
> oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
>    [junit4]   2>                commit{dir=MockDirectoryWrapper(
> RAMDirectory@6041646a lockFactory=org.apache.lucene.store.
> SingleInstanceLockFactory@65279c84),segFN=segments_1,generation=1}
>    [junit4]   2> 298700 T1299 C:control_collection c:collection1
> oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>    [junit4]   2> 298700 T1299 C:control_collection c:collection1
> oass.SolrIndexSearcher.<init> Opening Searcher@445242be[collection1] main
>    [junit4]   2> 298701 T1299 C:control_collection c:collection1
> oascc.ZkStateReader.readConfigName Load collection config
> from:/collections/control_collection
>    [junit4]   2> 298701 T1299 C:control_collection c:collection1
> oascc.ZkStateReader.readConfigName path=/collections/control_collection
> configName=conf1 specified config exists in ZooKeeper
>    [junit4]   2> 298701 T1299 C:control_collection c:collection1
> oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based
> storage for the RestManager with znodeBase: /configs/conf1
>    [junit4]   2> 298702 T1299 C:control_collection c:collection1
> oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured
> ZooKeeperStorageIO with znodeBase: /configs/conf1
>    [junit4]   2> 298702 T1299 C:control_collection c:collection1
> oasr.RestManager.init Initializing RestManager with initArgs: {}
>    [junit4]   2> 298702 T1299 C:control_collection c:collection1
> oasr.ManagedResourceStorage.load Reading _rest_managed.json using
> ZooKeeperStorageIO:path=/configs/conf1
>    [junit4]   2> 298703 T1299 C:control_collection c:collection1
> oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data
> found for znode /configs/conf1/_rest_managed.json
>    [junit4]   2> 298703 T1299 C:control_collection c:collection1
> oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json
> using ZooKeeperStorageIO:path=/configs/conf1
>    [junit4]   2> 298703 T1299 C:control_collection c:collection1
> oasr.RestManager.init Initializing 0 registered ManagedResources
>    [junit4]   2> 298703 T1299 C:control_collection c:collection1
> oash.ReplicationHandler.inform Commits will be reserved for  10000
>    [junit4]   2> 298704 T1299 C:control_collection c:collection1
> oasc.CoreContainer.registerCore registering core: collection1
>    [junit4]   2> 298704 T1300 C:control_collection c:collection1
> oasc.SolrCore.registerSearcher [collection1] Registered new searcher
> Searcher@445242be[collection1] main{ExitableDirectoryReader(
> UninvertingDirectoryReader())}
>    [junit4]   2> 298704 T1303 C:control_collection S:shard1
> oasc.ZkController.register Register replica - core:collection1 address:
> http://127.0.0.1:50635 collection:control_collection shard:shard1
>    [junit4]   2> 298705 T1269 oass.SolrDispatchFilter.init
> user.dir=/usr/home/jenkins/jenkins-slave/workspace/
> Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
>    [junit4]   2> 298706 T1269 oass.SolrDispatchFilter.init
> SolrDispatchFilter.init() done
>    [junit4]   2> 298706 T1303 C:control_collection S:shard1
> oascc.SolrZkClient.makePath makePath: /collections/control_
> collection/leader_elect/shard1/election
>    [junit4]   2> 298710 T1303 C:control_collection S:shard1 oasc.
> ShardLeaderElectionContext.runLeaderProcess Running the leader process
> for shard shard1
>    [junit4]   2> 298710 T1269 oascc.SolrZkClient.
> createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
>    [junit4]   2> 298711 T1295 oasc.DistributedQueue$LatchWatcher.process
> NodeChildrenChanged fired on path /overseer/queue state SyncConnected
>    [junit4]   2> 298711 T1303 C:control_collection S:shard1 oasc.
> ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found
> to continue.
>    [junit4]   2> 298711 T1269 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>    [junit4]   2> 298712 T1303 C:control_collection S:shard1 oasc.
> ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try
> and sync
>    [junit4]   2> ASYNC  NEW_CORE C279 name=collection1
> org.apache.solr.core.SolrCore@1745c55d url=http://127.0.0.1:50635/
> collection1 node=127.0.0.1:50635_ C279_STATE=coll:control_collection
> core:collection1 props:{state=down, base_url=http://127.0.0.1:50635,
> node_name=127.0.0.1:50635_, core=collection1}
>    [junit4]   2> 298712 T1303 C:control_collection S:shard1 C279 P50635
> oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:50635/
> collection1/
>    [junit4]   2> 298712 T1296 oasc.Overseer$ClusterStateUpdater.run
> processMessage: queueSize: 1, message = {
>    [junit4]   2>          "operation":"leader",
>    [junit4]   2>          "shard":"shard1",
>    [junit4]   2>          "collection":"control_collection"} current
> state version: 1
>    [junit4]   2> 298712 T1307 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@6ea67455
> name:ZooKeeperConnection Watcher:127.0.0.1:50623/solr got event
> WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 298712 T1303 C:control_collection S:shard1 C279 P50635
> oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
>    [junit4]   2> 298713 T1269 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>    [junit4]   2> 298713 T1303 C:control_collection S:shard1 C279 P50635
> oasc.SyncStrategy.syncToMe http://127.0.0.1:50635/collection1/ has no
> replicas
>    [junit4]   2> 298713 T1303 C:control_collection S:shard1 oasc.
> ShardLeaderElectionContext.runLeaderProcess I am the new leader:
> http://127.0.0.1:50635/collection1/ shard1
>    [junit4]   2> 298714 T1269 oascc.SolrZkClient.createZkACLProvider
> Using default ZkACLProvider
>    [junit4]   2> 298714 T1295 oascc.ZkStateReader$2.process A cluster
> state change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 298714 T1303 C:control_collection S:shard1
> oascc.SolrZkClient.makePath makePath: /collections/control_
> collection/leaders/shard1
>    [junit4]   2> 298714 T1269 oascc.ZkStateReader.
> createClusterStateWatchersAndUpdate Updating cluster state from
> ZooKeeper...
>    [junit4]   2> 298717 T1269 oasc.ChaosMonkey.monkeyLog monkey: init -
> expire sessions:false cause connection loss:false
>    [junit4]   2> 298718 T1269 oasc.AbstractFullDistribZkTestBase.createJettys
> Creating collection1 with stateFormat=2
>    [junit4]   2> 298719 T1269 oascc.SolrZkClient.
> createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
>    [junit4]   2> 298719 T1295 oasc.DistributedQueue$LatchWatcher.process
> NodeChildrenChanged fired on path /overseer/queue state SyncConnected
>    [junit4]   2> 298719 T1269 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>    [junit4]   2> 298720 T1296 oasc.Overseer$ClusterStateUpdater.run
> processMessage: queueSize: 1, message = {
>    [junit4]   2>          "operation":"leader",
>    [junit4]   2>          "shard":"shard1",
>    [junit4]   2>          "collection":"control_collection",
>    [junit4]   2>          "base_url":"http://127.0.0.1:50635",
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "state":"active"} current state version: 2
>    [junit4]   2> 298720 T1310 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@5c566433
> name:ZooKeeperConnection Watcher:127.0.0.1:50623/solr got event
> WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 298721 T1269 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>    [junit4]   2> 298721 T1269 oascc.SolrZkClient.createZkACLProvider
> Using default ZkACLProvider
>    [junit4]   2> 298722 T1295 oasc.DistributedQueue$LatchWatcher.process
> NodeChildrenChanged fired on path /overseer/queue state SyncConnected
>    [junit4]   2> 298723 T1296 oasc.Overseer$ClusterStateUpdater.run
> processMessage: queueSize: 1, message = {
>    [junit4]   2>          "operation":"create",
>    [junit4]   2>          "name":"collection1",
>    [junit4]   2>          "numShards":"1",
>    [junit4]   2>          "stateFormat":"2"} current state version: 2
>    [junit4]   2> 298723 T1296 oasco.ClusterStateMutator.createCollection
> building a new cName: collection1
>    [junit4]   2> 298724 T1295 oascc.ZkStateReader$2.process A cluster
> state change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 298724 T1307 oascc.ZkStateReader$2.process A cluster
> state change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 298769 T1303 C:control_collection S:shard1
> oasc.ZkController.register We are http://127.0.0.1:50635/collection1/ and
> leader is http://127.0.0.1:50635/collection1/
>    [junit4]   2> 298770 T1303 C:control_collection S:shard1
> oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=
> http://127.0.0.1:50635
>    [junit4]   2> 298770 T1303 C:control_collection S:shard1
> oasc.ZkController.checkRecovery I am the leader, no recovery necessary
>    [junit4]   2> 298770 T1303 C:control_collection S:shard1 c:collection1
> oasc.ZkController.publish publishing core=collection1 state=active
> collection=control_collection
>    [junit4]   2> 298770 T1303 C:control_collection S:shard1 c:collection1
> oasc.ZkController.publish numShards not found on descriptor - reading it
> from system property
>    [junit4]   2> 298771 T1295 oasc.DistributedQueue$LatchWatcher.process
> NodeChildrenChanged fired on path /overseer/queue state SyncConnected
>    [junit4]   2> 298772 T1296 oasc.Overseer$ClusterStateUpdater.run
> processMessage: queueSize: 1, message = {
>    [junit4]   2>          "collection":"control_collection",
>    [junit4]   2>          "core_node_name":"core_node1",
>    [junit4]   2>          "state":"active",
>    [junit4]   2>          "shard":"shard1",
>    [junit4]   2>          "base_url":"http://127.0.0.1:50635",
>    [junit4]   2>          "roles":null,
>    [junit4]   2>          "node_name":"127.0.0.1:50635_",
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "operation":"state",
>    [junit4]   2>          "numShards":"1"} current state version: 3
>    [junit4]   2> 298773 T1296 oasco.ReplicaMutator.updateState Update
> state numShards=1 message={
>    [junit4]   2>          "collection":"control_collection",
>    [junit4]   2>          "core_node_name":"core_node1",
>    [junit4]   2>          "state":"active",
>    [junit4]   2>          "shard":"shard1",
>    [junit4]   2>          "base_url":"http://127.0.0.1:50635",
>    [junit4]   2>          "roles":null,
>    [junit4]   2>          "node_name":"127.0.0.1:50635_",
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "operation":"state",
>    [junit4]   2>          "numShards":"1"}
>    [junit4]   2> 298773 T1296 oasco.ZkStateWriter.writePendingUpdates
> going to create_collection /collections/collection1/state.json
>    [junit4]   2> 298774 T1307 oascc.ZkStateReader$2.process A cluster
> state change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 298774 T1295 oascc.ZkStateReader$2.process A cluster
> state change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 298878 T1307 oascc.ZkStateReader$2.process A cluster
> state change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 298878 T1295 oascc.ZkStateReader$2.process A cluster
> state change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 300125 T1269 oas.SolrTestCaseJ4.writeCoreProperties
> Writing core.properties file to /usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/shard-1-001/cores/collection1
>    [junit4]   2> 300128 T1269 oasc.AbstractFullDistribZkTestBase.createJettys
> create jetty 1 in directory /usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/shard-1-001
>    [junit4]   2> 300128 T1269 oejs.Server.doStart jetty-8.1.10.v20130312
>    [junit4]   2> 300132 T1269 oejs.AbstractConnector.doStart Started
> SocketConnector@127.0.0.1:50644
>    [junit4]   2> 300132 T1269 oascse.JettySolrRunner$1.lifeCycleStarted
> Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/
> workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/
> test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/shard-1-001/cores, solr.data.dir=/usr/home/
> jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.
> x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/tempDir-001/jetty1, hostContext=/,
> solrconfig=solrconfig.xml, hostPort=50643}
>    [junit4]   2> 300133 T1269 oass.SolrDispatchFilter.init
> SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
>    [junit4]   2> 300133 T1269 oasc.SolrResourceLoader.<init> new
> SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/shard-1-001/'
>    [junit4]   2> 300171 T1269 oasc.SolrXmlConfig.fromFile Loading
> container configuration from /usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/shard-1-001/solr.xml
>    [junit4]   2> 300213 T1269 oasc.CorePropertiesLocator.<init>
> Config-defined core root directory: /usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/shard-1-001/cores
>    [junit4]   2> 300213 T1269 oasc.CoreContainer.<init> New CoreContainer
> 1370575573
>    [junit4]   2> 300214 T1269 oasc.CoreContainer.load Loading cores into
> CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/
> workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/
> test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/shard-1-001/]
>    [junit4]   2> 300214 T1269 oasc.CoreContainer.load loading shared
> library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/shard-1-001/lib
>    [junit4]   2> 300215 T1269 oasc.SolrResourceLoader.addToClassLoader
> WARN Can

[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2900 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2900/

4 tests failed.
REGRESSION:  org.apache.solr.cloud.RecoveryZkTest.test

Error Message:
shard1 is not consistent.  Got 1023 from http://127.0.0.1:10050/p_/z/collection1lastClient and got 295 from http://127.0.0.1:10059/p_/z/collection1

Stack Trace:
java.lang.AssertionError: shard1 is not consistent.  Got 1023 from http://127.0.0.1:10050/p_/z/collection1lastClient and got 295 from http://127.0.0.1:10059/p_/z/collection1
	at __randomizedtesting.SeedInfo.seed([FBFBECE5D5AABA29:73AFD33F7B56D7D1]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:123)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:52540/fmq/gz/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:52540/fmq/gz/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([FBFBECE5D5AABA29:73AFD33F7B56D7D1]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:52540 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50643/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50643/collection1
	at __randomizedtesting.SeedInfo.seed([FBFBECE5D5AABA29:73AFD33F7B56D7D1]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50643/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:50643 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:11223/repfacttest_c8n_1x3_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:11223/repfacttest_c8n_1x3_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([FBFBECE5D5AABA29:73AFD33F7B56D7D1]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:11223/repfacttest_c8n_1x3_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:11223 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9494 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/init-core-data-001
   [junit4]   2> 295434 T1269 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 295441 T1269 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 295441 T1270 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 295541 T1269 oasc.ZkTestServer.run start zk server on port:50623
   [junit4]   2> 295542 T1269 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 295542 T1269 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 295545 T1277 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6e5605f4 name:ZooKeeperConnection Watcher:127.0.0.1:50623 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 295545 T1269 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 295546 T1269 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 295546 T1269 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 295548 T1269 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 295549 T1269 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 295550 T1280 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@842caa6 name:ZooKeeperConnection Watcher:127.0.0.1:50623/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 295550 T1269 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 295551 T1269 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 295551 T1269 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 295553 T1269 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 295554 T1269 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 295555 T1269 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 295556 T1269 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 295557 T1269 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 295559 T1269 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 295560 T1269 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 295661 T1269 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 295662 T1269 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 295663 T1269 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 295663 T1269 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 295665 T1269 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 295665 T1269 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 295667 T1269 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 295667 T1269 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 295668 T1269 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 295669 T1269 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 295670 T1269 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 295670 T1269 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 295672 T1269 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 295672 T1269 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 295673 T1269 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 295674 T1269 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 295675 T1269 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 295675 T1269 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 296963 T1269 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores/collection1
   [junit4]   2> 296967 T1269 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 296971 T1269 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:50636
   [junit4]   2> 296971 T1269 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=50635, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/tempDir-001/control/data}
   [junit4]   2> 296971 T1269 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 296972 T1269 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/'
   [junit4]   2> 297015 T1269 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/solr.xml
   [junit4]   2> 297067 T1269 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores
   [junit4]   2> 297069 T1269 oasc.CoreContainer.<init> New CoreContainer 2110879613
   [junit4]   2> 297069 T1269 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/]
   [junit4]   2> 297069 T1269 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/lib
   [junit4]   2> 297070 T1269 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/lib).
   [junit4]   2> 297080 T1269 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 297080 T1269 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 297080 T1269 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 297080 T1269 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 297081 T1269 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 297081 T1269 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 297081 T1269 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 297081 T1269 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 297082 T1269 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 297082 T1269 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 297082 T1269 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 297089 T1269 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 297090 T1269 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 297090 T1269 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 297090 T1269 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 297091 T1269 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50623/solr
   [junit4]   2> 297091 T1269 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 297092 T1269 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 297092 T1269 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 297094 T1292 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@45cc06e8 name:ZooKeeperConnection Watcher:127.0.0.1:50623 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 297095 T1269 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 297096 T1269 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 297097 T1269 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 297098 T1295 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@34504d13 name:ZooKeeperConnection Watcher:127.0.0.1:50623/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 297098 T1269 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 297100 T1269 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 297101 T1269 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 297103 T1269 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 297104 T1269 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 297105 T1269 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 297107 T1269 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 297108 T1269 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 297109 T1269 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 297110 T1269 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50635_
   [junit4]   2> 297110 T1269 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50635_
   [junit4]   2> 297112 T1269 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 297113 T1269 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 297114 T1269 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 297115 T1269 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:50635_
   [junit4]   2> 297115 T1269 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 297116 T1269 oasc.Overseer.start Overseer (id=93607700461191171-127.0.0.1:50635_-n_0000000000) starting
   [junit4]   2> 297118 T1269 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 297122 T1269 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 297122 T1297 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 297122 T1269 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 297122 T1296 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 297126 T1269 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores
   [junit4]   2> 297128 T1269 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, collection=control_collection, coreNodeName=, dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores/collection1/, config=solrconfig.xml, schema=schema.xml, loadOnStartup=true, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores/collection1, name=collection1}
   [junit4]   2> 297128 T1269 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores/collection1/
   [junit4]   2> 297128 T1269 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 297130 T1299 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 297130 T1299 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 297131 T1299 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 297131 T1295 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 297132 T1296 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50635",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50635_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"} current state version: 0
   [junit4]   2> 297132 T1296 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50635",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50635_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 297132 T1296 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 297133 T1296 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 297134 T1295 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 298131 T1299 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 298132 T1299 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 298132 T1299 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 298132 T1299 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 298133 T1299 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 298133 T1299 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores/collection1/'
   [junit4]   2> 298147 T1299 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 298149 T1299 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 298156 T1299 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 298172 T1299 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 298210 T1299 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 298311 T1299 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 298318 T1299 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 298623 T1299 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 298636 T1299 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 298638 T1299 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 298648 T1299 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 298651 T1299 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 298654 T1299 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 298655 T1299 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 298655 T1299 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 298655 T1299 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 298656 T1299 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 298656 T1299 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 298656 T1299 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 298656 T1299 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 298657 T1299 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores/collection1/data/
   [junit4]   2> 298657 T1299 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ec2b6ef
   [junit4]   2> 298658 T1299 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores/collection1/data
   [junit4]   2> 298658 T1299 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores/collection1/data/index/
   [junit4]   2> 298658 T1299 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 298658 T1299 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores/collection1/data/index
   [junit4]   2> 298659 T1299 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=27, maxMergeAtOnceExplicit=50, maxMergedSegmentMB=52.0615234375, floorSegmentMB=0.486328125, forceMergeDeletesPctAllowed=10.112383767408401, segmentsPerTier=19.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 298659 T1299 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6041646a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@65279c84),segFN=segments_1,generation=1}
   [junit4]   2> 298659 T1299 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 298664 T1299 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 298664 T1299 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 298664 T1299 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 298664 T1299 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 298665 T1299 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 298665 T1299 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 298665 T1299 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 298665 T1299 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 298666 T1299 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 298685 T1299 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 298687 T1299 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 298689 T1299 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 298690 T1299 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 298694 T1299 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /schema,/update/json/docs,/admin/threads,/config,/replication,/admin/file,/admin/logging,/admin/plugins,/update/csv,standard,/admin/ping,/admin/system,/get,/admin/luke,/admin/segments,/admin/mbeans,/admin/properties,/update,/update/json
   [junit4]   2> 298696 T1299 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 298697 T1299 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 298697 T1299 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 298699 T1299 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 298699 T1299 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 298699 T1299 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=39.86328125, floorSegmentMB=2.154296875, forceMergeDeletesPctAllowed=25.115751995354497, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 298700 T1299 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6041646a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@65279c84),segFN=segments_1,generation=1}
   [junit4]   2> 298700 T1299 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 298700 T1299 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@445242be[collection1] main
   [junit4]   2> 298701 T1299 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 298701 T1299 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 298701 T1299 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 298702 T1299 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 298702 T1299 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 298702 T1299 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 298703 T1299 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 298703 T1299 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 298703 T1299 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 298703 T1299 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 298704 T1299 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 298704 T1300 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@445242be[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 298704 T1303 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50635 collection:control_collection shard:shard1
   [junit4]   2> 298705 T1269 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 298706 T1269 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 298706 T1303 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 298710 T1303 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 298710 T1269 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 298711 T1295 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 298711 T1303 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 298711 T1269 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 298712 T1303 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C279 name=collection1 org.apache.solr.core.SolrCore@1745c55d url=http://127.0.0.1:50635/collection1 node=127.0.0.1:50635_ C279_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:50635, node_name=127.0.0.1:50635_, core=collection1}
   [junit4]   2> 298712 T1303 C:control_collection S:shard1 C279 P50635 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:50635/collection1/
   [junit4]   2> 298712 T1296 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 298712 T1307 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6ea67455 name:ZooKeeperConnection Watcher:127.0.0.1:50623/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 298712 T1303 C:control_collection S:shard1 C279 P50635 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 298713 T1269 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 298713 T1303 C:control_collection S:shard1 C279 P50635 oasc.SyncStrategy.syncToMe http://127.0.0.1:50635/collection1/ has no replicas
   [junit4]   2> 298713 T1303 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:50635/collection1/ shard1
   [junit4]   2> 298714 T1269 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 298714 T1295 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 298714 T1303 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 298714 T1269 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 298717 T1269 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 298718 T1269 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 298719 T1269 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 298719 T1295 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 298719 T1269 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 298720 T1296 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50635",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 298720 T1310 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5c566433 name:ZooKeeperConnection Watcher:127.0.0.1:50623/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 298721 T1269 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 298721 T1269 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 298722 T1295 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 298723 T1296 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 298723 T1296 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 298724 T1295 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 298724 T1307 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 298769 T1303 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:50635/collection1/ and leader is http://127.0.0.1:50635/collection1/
   [junit4]   2> 298770 T1303 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:50635
   [junit4]   2> 298770 T1303 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 298770 T1303 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 298770 T1303 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 298771 T1295 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 298772 T1296 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50635",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50635_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"} current state version: 3
   [junit4]   2> 298773 T1296 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50635",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50635_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 298773 T1296 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 298774 T1307 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 298774 T1295 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 298878 T1307 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 298878 T1295 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 300125 T1269 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores/collection1
   [junit4]   2> 300128 T1269 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001
   [junit4]   2> 300128 T1269 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 300132 T1269 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:50644
   [junit4]   2> 300132 T1269 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/tempDir-001/jetty1, hostContext=/, solrconfig=solrconfig.xml, hostPort=50643}
   [junit4]   2> 300133 T1269 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 300133 T1269 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/'
   [junit4]   2> 300171 T1269 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/solr.xml
   [junit4]   2> 300213 T1269 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores
   [junit4]   2> 300213 T1269 oasc.CoreContainer.<init> New CoreContainer 1370575573
   [junit4]   2> 300214 T1269 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/]
   [junit4]   2> 300214 T1269 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/lib
   [junit4]   2> 300215 T1269 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/lib).
   [junit4]   2> 300226 T1269 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 300226 T1269 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 300227 T1269 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 300227 T1269 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 300227 T1269 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 300227 T1269 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 300228 T1269 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 300228 T1269 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 300228 T1269 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 300228 T1269 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 300229 T1269 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 300234 T1269 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 300235 T1269 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 300235 T1269 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 300236 T1269 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 300236 T1269 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50623/solr
   [junit4]   2> 300236 T1269 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 300236 T1269 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 300237 T1269 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 300239 T1322 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@15d18950 name:ZooKeeperConnection Watcher:127.0.0.1:50623 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 300240 T1269 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 300242 T1269 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 300243 T1269 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 300245 T1325 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7a93df95 name:ZooKeeperConnection Watcher:127.0.0.1:50623/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 300245 T1269 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 300248 T1269 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 301258 T1269 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50643_
   [junit4]   2> 301259 T1269 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50643_
   [junit4]   2> 301262 T1269 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 301267 T1269 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores
   [junit4]   2> 301269 T1269 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores/collection1/, name=collection1, shard=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores/collection1, config=solrconfig.xml, loadOnStartup=true, transient=false, coreNodeName=, dataDir=data/, schema=schema.xml}
   [junit4]   2> 301270 T1269 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores/collection1/
   [junit4]   2> 301270 T1269 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 301273 T1326 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 301273 T1326 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 301275 T1295 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 301275 T1326 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 301275 T1326 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 301275 T1296 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50643",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50643_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"} current state version: 5
   [junit4]   2> 301276 T1326 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 301276 T1296 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50643",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50643_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 301276 T1296 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 301276 T1296 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 301277 T1326 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 301277 T1326 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 301278 T1296 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 301278 T1325 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 301279 T1325 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 302277 T1326 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 302278 T1326 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 302281 T1326 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 302282 T1326 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 302283 T1326 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 302284 T1326 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores/collection1/'
   [junit4]   2> 302320 T1326 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 302325 T1326 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 302356 T1326 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 302401 T1326 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 302535 T1326 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 302639 T1326 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 302662 T1326 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 303585 T1326 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 303613 T1326 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 303616 T1326 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 303637 T1326 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 303645 T1326 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 303652 T1326 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 303653 T1326 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 303654 T1326 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 303654 T1326 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 303655 T1326 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 303655 T1326 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 303655 T1326 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 303656 T1326 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 303656 T1326 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 303657 T1326 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ec2b6ef
   [junit4]   2> 303658 T1326 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores/collection1/data
   [junit4]   2> 303658 T1326 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 303658 T1326 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 303659 T1326 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 303659 T1326 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=27, maxMergeAtOnceExplicit=50, maxMergedSegmentMB=52.0615234375, floorSegmentMB=0.486328125, forceMergeDeletesPctAllowed=10.112383767408401, segmentsPerTier=19.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 303661 T1326 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5be0b2e2 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7d00428d),segFN=segments_1,generation=1}
   [junit4]   2> 303661 T1326 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 303669 T1326 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 303669 T1326 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 303670 T1326 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 303670 T1326 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 303670 T1326 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 303671 T1326 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 303671 T1326 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 303671 T1326 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 303672 T1326 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 303704 T1326 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 303707 T1326 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 303710 T1326 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 303713 T1326 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 303719 T1326 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /schema,/update/json/docs,/admin/threads,/config,/replication,/admin/file,/admin/logging,/admin/plugins,/update/csv,standard,/admin/ping,/admin/system,/get,/admin/luke,/admin/segments,/admin/mbeans,/admin/properties,/update,/update/json
   [junit4]   2> 303722 T1326 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 303724 T1326 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 303724 T1326 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 303726 T1326 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 303726 T1326 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 303727 T1326 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=39.86328125, floorSegmentMB=2.154296875, forceMergeDeletesPctAllowed=25.115751995354497, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 303728 T1326 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5be0b2e2 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7d00428d),segFN=segments_1,generation=1}
   [junit4]   2> 303728 T1326 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 303729 T1326 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@83d513f[collection1] main
   [junit4]   2> 303729 T1326 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 303730 T1326 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 303730 T1326 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 303731 T1326 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 303731 T1326 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 303731 T1326 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 303732 T1326 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 303732 T1326 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 303732 T1326 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 303733 T1326 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 303734 T1327 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@83d513f[collection1] main{Exi

[...truncated too long message...]

96/
   [junit4]   2> 1572522 T4874 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:11212/
   [junit4]   2> 1572522 T4874 C:control_collection S:shard1 c:collection1 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1572522 T4874 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:11208/
   [junit4]   2> 1572522 T4874 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:11223/
   [junit4]   2> 1572523 T4874 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:11196/
   [junit4]   2> 1572523 T4874 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:11212/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=FBFBECE5D5AABA29 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=es_ES -Dtests.timezone=America/Rankin_Inlet -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   73.2s J0 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:11223/repfacttest_c8n_1x3_shard1_replica2
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([FBFBECE5D5AABA29:73AFD33F7B56D7D1]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:11223/repfacttest_c8n_1x3_shard1_replica2
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:11223 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 53 more
   [junit4]   2> 1572544 T4874 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest FBFBECE5D5AABA29-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {range_facet_l=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), intDefault=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), text=PostingsFormat(name=Memory doPackFST= false), _version_=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), timestamp=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), multiDefault=FSTOrd50, a_t=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), id=Lucene50(blocksize=128)}, docValues:{timestamp=DocValuesFormat(name=Direct)}, sim=RandomSimilarityProvider(queryNorm=false,coord=crazy): {}, locale=es_ES, timezone=America/Rankin_Inlet
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=161230816,total=342884352
   [junit4]   2> NOTE: All tests run in this JVM: [TestIntervalFaceting, XmlUpdateRequestHandlerTest, CoreAdminRequestStatusTest, TestReplicaProperties, TestJettySolrRunner, TestRestManager, TestTrie, CollectionReloadTest, HardAutoCommitTest, MinimalSchemaTest, WordBreakSolrSpellCheckerTest, PolyFieldTest, TestHdfsUpdateLog, TestCursorMarkWithoutUniqueKey, QueryEqualityTest, OverseerRolesTest, TestQueryUtils, DirectUpdateHandlerOptimizeTest, TestFreeTextSuggestions, TriLevelCompositeIdRoutingTest, SpellPossibilityIteratorTest, TestAddFieldRealTimeGet, TestDFRSimilarityFactory, DeleteShardTest, PreAnalyzedUpdateProcessorTest, TestSchemaResource, HdfsSyncSliceTest, ZkCLITest, ShowFileRequestHandlerTest, TestQuerySenderListener, RequiredFieldsTest, SoftAutoCommitTest, TestIndexingPerformance, TestCloudPivotFacet, TestManagedSchemaFieldResource, TestSimpleTrackingShardHandler, TestLFUCache, CustomCollectionTest, TestInitQParser, DistributedQueryComponentCustomSortTest, MBeansHandlerTest, TimeZoneUtilsTest, SolrRequestParserTest, SolrTestCaseJ4Test, TestShardHandlerFactory, FileUtilsTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, ZkNodePropsTest, TestCSVResponseWriter, TestManagedStopFilterFactory, CacheHeaderTest, SolrCmdDistributorTest, TestConfig, TestBM25SimilarityFactory, TestManagedResource, MultiThreadedOCPTest, SyncSliceTest, TestCloudManagedSchemaConcurrent, ChaosMonkeyNothingIsSafeTest, ReplicationFactorTest]
   [junit4] Completed on J0 in 73.22s, 1 test, 1 error <<< FAILURES!

[...truncated 698 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 479 suites, 1903 tests, 3 errors, 1 failure, 61 ignored (21 assumptions)

Total time: 53 minutes 7 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 218990361 bytes
Compression is 0.0%
Took 40 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2899 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2899/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:37382/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37382/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([47F5277B5BCEDC6F:CFA118A1F532B197]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:37382 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22460/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22460/collection1
	at __randomizedtesting.SeedInfo.seed([47F5277B5BCEDC6F:CFA118A1F532B197]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22460/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:22460 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16215/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16215/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([47F5277B5BCEDC6F:CFA118A1F532B197]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16215/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:16215 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9841 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/init-core-data-001
   [junit4]   2> 653060 T1976 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 653067 T1976 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 653068 T1977 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 653168 T1976 oasc.ZkTestServer.run start zk server on port:22447
   [junit4]   2> 653169 T1976 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 653170 T1976 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 653174 T1984 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@110ad99a name:ZooKeeperConnection Watcher:127.0.0.1:22447 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 653174 T1976 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 653174 T1976 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 653175 T1976 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 653177 T1976 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 653178 T1976 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 653179 T1987 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@30950d04 name:ZooKeeperConnection Watcher:127.0.0.1:22447/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 653179 T1976 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 653180 T1976 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 653180 T1976 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 653182 T1976 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 653183 T1976 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 653184 T1976 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 653186 T1976 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 653186 T1976 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 653189 T1976 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 653189 T1976 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 653291 T1976 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 653291 T1976 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 653293 T1976 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 653294 T1976 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 653295 T1976 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 653296 T1976 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 653298 T1976 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 653298 T1976 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 653300 T1976 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 653300 T1976 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 653302 T1976 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 653302 T1976 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 653304 T1976 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 653304 T1976 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 653306 T1976 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 653307 T1976 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 653308 T1976 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 653309 T1976 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 654775 T1976 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/cores/collection1
   [junit4]   2> 654779 T1976 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 654781 T1976 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:22455
   [junit4]   2> 654782 T1976 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=22454, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/tempDir-001/control/data}
   [junit4]   2> 654783 T1976 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 654783 T1976 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/'
   [junit4]   2> 654821 T1976 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/solr.xml
   [junit4]   2> 654863 T1976 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/cores
   [junit4]   2> 654863 T1976 oasc.CoreContainer.<init> New CoreContainer 594929581
   [junit4]   2> 654863 T1976 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/]
   [junit4]   2> 654864 T1976 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/lib
   [junit4]   2> 654864 T1976 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/lib).
   [junit4]   2> 654874 T1976 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 654874 T1976 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 654875 T1976 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 654875 T1976 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 654875 T1976 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 654875 T1976 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 654876 T1976 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 654876 T1976 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 654876 T1976 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 654876 T1976 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 654876 T1976 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 654882 T1976 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 654883 T1976 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 654883 T1976 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 654883 T1976 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 654884 T1976 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:22447/solr
   [junit4]   2> 654884 T1976 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 654884 T1976 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 654885 T1976 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 654886 T1999 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@646e9be3 name:ZooKeeperConnection Watcher:127.0.0.1:22447 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 654887 T1976 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 654888 T1976 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 654889 T1976 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 654890 T2002 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1c3dae1f name:ZooKeeperConnection Watcher:127.0.0.1:22447/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 654890 T1976 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 654891 T1976 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 654893 T1976 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 654894 T1976 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 654895 T1976 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 654897 T1976 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 654898 T1976 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 654900 T1976 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 654901 T1976 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 654901 T1976 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:22454_
   [junit4]   2> 654902 T1976 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:22454_
   [junit4]   2> 654903 T1976 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 654904 T1976 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 654905 T1976 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 654906 T1976 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:22454_
   [junit4]   2> 654906 T1976 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 654908 T1976 oasc.Overseer.start Overseer (id=93606581219819523-127.0.0.1:22454_-n_0000000000) starting
   [junit4]   2> 654909 T1976 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 654914 T1976 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 654914 T2004 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 654914 T1976 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 654914 T2003 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 654917 T1976 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/cores
   [junit4]   2> 654919 T1976 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, coreNodeName=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/cores/collection1/, dataDir=data/, transient=false, collection=control_collection, loadOnStartup=true, config=solrconfig.xml, shard=, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/cores/collection1}
   [junit4]   2> 654919 T1976 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/cores/collection1/
   [junit4]   2> 654919 T1976 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 654920 T2006 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 654920 T2006 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 654921 T2006 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 654921 T2002 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 654922 T2003 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:22454_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22454",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down"} current state version: 0
   [junit4]   2> 654922 T2003 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:22454_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22454",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 654922 T2003 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 654923 T2003 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 654924 T2002 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 655921 T2006 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 655922 T2006 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 655922 T2006 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 655923 T2006 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 655923 T2006 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 655923 T2006 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/cores/collection1/'
   [junit4]   2> 655937 T2006 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 655939 T2006 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 655948 T2006 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 655966 T2006 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 656004 T2006 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 656105 T2006 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 656113 T2006 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 656566 T2006 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 656582 T2006 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 656585 T2006 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 656596 T2006 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 656600 T2006 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 656604 T2006 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 656605 T2006 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 656605 T2006 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 656606 T2006 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 656606 T2006 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 656606 T2006 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 656607 T2006 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 656607 T2006 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 656608 T2006 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/cores/collection1/data/
   [junit4]   2> 656608 T2006 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@405600d7
   [junit4]   2> 656609 T2006 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/cores/collection1/data
   [junit4]   2> 656609 T2006 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/cores/collection1/data/index/
   [junit4]   2> 656609 T2006 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 656611 T2006 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/cores/collection1/data/index
   [junit4]   2> 656611 T2006 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=50, maxMergeAtOnceExplicit=20, maxMergedSegmentMB=80.2119140625, floorSegmentMB=1.5, forceMergeDeletesPctAllowed=3.3180332404560553, segmentsPerTier=14.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1350957791224503
   [junit4]   2> 656612 T2006 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@62a9cba0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7977a7fa),segFN=segments_1,generation=1}
   [junit4]   2> 656612 T2006 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 656619 T2006 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 656619 T2006 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 656620 T2006 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 656620 T2006 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 656620 T2006 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 656620 T2006 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 656621 T2006 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 656621 T2006 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 656622 T2006 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 656644 T2006 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 656646 T2006 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 656649 T2006 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 656651 T2006 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 656657 T2006 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/json/docs,/admin/luke,/config,standard,/admin/segments,/replication,/admin/system,/get,/admin/threads,/admin/plugins,/admin/file,/schema,/update,/admin/mbeans,/admin/logging,/update/json,/admin/ping,/admin/properties,/update/csv
   [junit4]   2> 656659 T2006 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 656661 T2006 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 656661 T2006 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 656663 T2006 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 656663 T2006 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 656664 T2006 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=14, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.33693801321823635]
   [junit4]   2> 656666 T2006 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@62a9cba0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7977a7fa),segFN=segments_1,generation=1}
   [junit4]   2> 656666 T2006 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 656667 T2006 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@3f3cc2fa[collection1] main
   [junit4]   2> 656667 T2006 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 656668 T2006 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 656668 T2006 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 656669 T2006 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 656669 T2006 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 656669 T2006 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 656670 T2006 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 656670 T2006 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 656671 T2006 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 656671 T2006 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 656672 T2007 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3f3cc2fa[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 656672 T2006 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 656673 T2010 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:22454 collection:control_collection shard:shard1
   [junit4]   2> 656674 T1976 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 656674 T1976 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 656674 T2010 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 656678 T1976 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 656678 T1976 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 656678 T2010 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 656680 T2014 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2a5a5614 name:ZooKeeperConnection Watcher:127.0.0.1:22447/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 656680 T1976 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 656680 T2002 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 656681 T1976 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 656681 T2010 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 656681 T1976 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 656681 T2010 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 656681 T2003 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C227 name=collection1 org.apache.solr.core.SolrCore@49048db8 url=http://127.0.0.1:22454/collection1 node=127.0.0.1:22454_ C227_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:22454_, base_url=http://127.0.0.1:22454, core=collection1, state=down}
   [junit4]   2> 656681 T2010 C:control_collection S:shard1 C227 P22454 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:22454/collection1/
   [junit4]   2> 656682 T2010 C:control_collection S:shard1 C227 P22454 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 656682 T2010 C:control_collection S:shard1 C227 P22454 oasc.SyncStrategy.syncToMe http://127.0.0.1:22454/collection1/ has no replicas
   [junit4]   2> 656682 T2010 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:22454/collection1/ shard1
   [junit4]   2> 656682 T2010 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 656683 T2002 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 656683 T2014 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 656684 T1976 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 656684 T1976 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 656685 T1976 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 656685 T1976 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 656687 T2017 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3ced76f4 name:ZooKeeperConnection Watcher:127.0.0.1:22447/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 656687 T1976 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 656687 T2002 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 656687 T1976 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 656688 T2003 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22454",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 656690 T2003 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 656691 T2003 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 656691 T2014 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 656691 T2002 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 656694 T2002 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 656695 T2003 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 656696 T2002 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 656696 T2014 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 656738 T2010 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:22454/collection1/ and leader is http://127.0.0.1:22454/collection1/
   [junit4]   2> 656739 T2010 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:22454
   [junit4]   2> 656739 T2010 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 656739 T2010 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 656739 T2010 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 656741 T2002 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 656742 T2003 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:22454_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22454",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active"} current state version: 4
   [junit4]   2> 656742 T2003 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:22454_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22454",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active"}
   [junit4]   2> 656846 T2002 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 656846 T2014 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 658577 T1976 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/cores/collection1
   [junit4]   2> 658581 T1976 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001
   [junit4]   2> 658581 T1976 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 658584 T1976 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:22461
   [junit4]   2> 658584 T1976 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=22460, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/cores}
   [junit4]   2> 658585 T1976 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 658585 T1976 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/'
   [junit4]   2> 658629 T1976 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/solr.xml
   [junit4]   2> 658684 T1976 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/cores
   [junit4]   2> 658684 T1976 oasc.CoreContainer.<init> New CoreContainer 1807901597
   [junit4]   2> 658685 T1976 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/]
   [junit4]   2> 658685 T1976 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/lib
   [junit4]   2> 658686 T1976 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/lib).
   [junit4]   2> 658697 T1976 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 658698 T1976 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 658698 T1976 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 658698 T1976 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 658698 T1976 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 658698 T1976 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 658699 T1976 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 658699 T1976 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 658699 T1976 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 658699 T1976 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 658700 T1976 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 658708 T1976 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 658708 T1976 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 658709 T1976 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 658709 T1976 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 658709 T1976 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:22447/solr
   [junit4]   2> 658709 T1976 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 658710 T1976 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 658710 T1976 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 658712 T2029 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@71de2711 name:ZooKeeperConnection Watcher:127.0.0.1:22447 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 658713 T1976 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 658714 T1976 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 658715 T1976 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 658717 T2032 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@520ccf14 name:ZooKeeperConnection Watcher:127.0.0.1:22447/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 658717 T1976 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 658720 T1976 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 659724 T1976 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:22460_
   [junit4]   2> 659725 T1976 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:22460_
   [junit4]   2> 659727 T1976 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 659730 T1976 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/cores
   [junit4]   2> 659732 T1976 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {coreNodeName=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/cores/collection1/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/cores/collection1, collection=collection1, dataDir=data/, shard=, loadOnStartup=true, transient=false, config=solrconfig.xml, name=collection1, schema=schema.xml}
   [junit4]   2> 659733 T1976 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/cores/collection1/
   [junit4]   2> 659733 T1976 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 659735 T2033 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 659735 T2033 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 659736 T2002 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 659737 T2033 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 659737 T2003 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:22460_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22460",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down"} current state version: 5
   [junit4]   2> 659737 T2033 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 659738 T2003 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:22460_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22460",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 659738 T2033 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 659738 T2003 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 659738 T2003 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 659739 T2033 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 659739 T2033 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 659739 T2003 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 659740 T2032 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 659740 T2032 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 660739 T2033 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 660740 T2033 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 660741 T2033 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 660741 T2033 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 660742 T2033 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 660742 T2033 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/cores/collection1/'
   [junit4]   2> 660760 T2033 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 660763 T2033 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 660775 T2033 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 660799 T2033 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 660853 T2033 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 660954 T2033 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 660961 T2033 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 661256 T2033 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 661277 T2033 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 661280 T2033 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 661288 T2033 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 661292 T2033 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 661295 T2033 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 661296 T2033 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 661296 T2033 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 661297 T2033 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 661297 T2033 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 661297 T2033 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 661298 T2033 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 661298 T2033 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 661298 T2033 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 661298 T2033 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@405600d7
   [junit4]   2> 661299 T2033 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/cores/collection1/data
   [junit4]   2> 661299 T2033 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 661300 T2033 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 661300 T2033 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 661300 T2033 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=50, maxMergeAtOnceExplicit=20, maxMergedSegmentMB=80.2119140625, floorSegmentMB=1.5, forceMergeDeletesPctAllowed=3.3180332404560553, segmentsPerTier=14.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1350957791224503
   [junit4]   2> 661301 T2033 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3d4a4473 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@56d3f974),segFN=segments_1,generation=1}
   [junit4]   2> 661301 T2033 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 661307 T2033 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 661307 T2033 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 661307 T2033 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 661308 T2033 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 661308 T2033 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 661308 T2033 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 661308 T2033 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 661309 T2033 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 661309 T2033 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 661330 T2033 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 661332 T2033 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 661334 T2033 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 661336 T2033 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 661342 T2033 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/json/docs,/admin/luke,/config,standard,/admin/segments,/replication,/admin/system,/get,/admin/threads,/admin/plugins,/admin/file,/schema,/update,/admin/mbeans,/admin/logging,/update/json,/admin/ping,/admin/properties,/update/csv
   [junit4]   2> 661344 T2033 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 661345 T2033 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 661345 T2033 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 661347 T2033 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 661347 T2033 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 661348 T2033 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=14, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.33693801321823635]
   [junit4]   2> 661349 T2033 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3d4a4473 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@56d3f974),segFN=segments_1,generation=1}
   [junit4]   2> 661349 T2033 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 661349 T2033 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@52bd8264[collection1] main
   [junit4]   2> 661349 T2033 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 661350 T2033 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 661350 T2033 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 661350 T2033 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 661351 T2033 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 661351 T2033 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 661351 T2033 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 661351 T2033 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 661352 T2033 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 661352 T2033 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 661353 T2034 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@52bd8264[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 661353 T2033 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 661354 T2037 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:22460 collection:collection1 shard:shard1
   [junit4]   2> 661354 T1976 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 661354 T2037 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 661355 T1976 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 661358 T2037 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 661359 T2002 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 661359 T2037 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 661359 T2037 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C228 name=collection1 org.apache.solr.core.SolrCore@eba0df9 url=http://127.0.0.1:22460/collection1 node=127.0.0.1:22460_ C228_STATE=coll:collection1 core:collection1 props:{node_name=127.0.0.1:22460_, base_url=http://127.0.0.1:22460, core=collection1, state=down}
   [junit4]   2> 661360 T2037 C:collection1 S:shard1 C228 P22460 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:22460/collection1/
   [junit4]   2> 661360 T2003 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 661360 T2037 C:collection1 S:shard1 C228 P22460 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 661360 T2037 C:collection1 S:shard1 C228 P22460 oasc.SyncStrategy.syncToMe http://127.0.0.1:22460/collection1/ has no replicas
   [junit4]   2> 661360 T2037 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:22460/collection1/ shard1
   [junit4]   2> 661360 T2037 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 661361 T2003 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 661361 T2032 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 661362 T2032 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 661364 T2002 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 661364 T2003 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22460",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 661365 T2003 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 661366 T2032 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 661366 T2032 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 661414 T2037 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:22460/collection1/ and leader is http://127.0.0.1:22460/collection1/
   [junit4]   2> 661415 T2037 C:collection1 S:shard1 

[...truncated too long message...]

5 T6359 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 2 connections to: http://127.0.0.1:16257/
   [junit4]   2> 1519576 T6359 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:16228/
   [junit4]   2> 1519576 T6359 C:control_collection S:shard1 c:collection1 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1519576 T6359 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16215/
   [junit4]   2> 1519577 T6359 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16264/
   [junit4]   2> 1519577 T6359 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16257/
   [junit4]   2> 1519577 T6359 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16228/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=47F5277B5BCEDC6F -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=is_IS -Dtests.timezone=America/Kralendijk -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   82.0s J3 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16215/repfacttest_c8n_1x3_shard1_replica3
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([47F5277B5BCEDC6F:CFA118A1F532B197]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16215/repfacttest_c8n_1x3_shard1_replica3
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:16215 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 53 more
   [junit4]   2> 1519603 T6359 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 47F5277B5BCEDC6F-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {timestamp=Lucene50(blocksize=128), intDefault=PostingsFormat(name=MockRandom), a_t=Lucene50(blocksize=128), range_facet_l=Lucene50(blocksize=128), id=PostingsFormat(name=LuceneVarGapFixedInterval), text=PostingsFormat(name=LuceneVarGapDocFreqInterval), _version_=PostingsFormat(name=MockRandom), multiDefault=Lucene50(blocksize=128)}, docValues:{timestamp=DocValuesFormat(name=Memory)}, sim=DefaultSimilarity, locale=is_IS, timezone=America/Kralendijk
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=172440416,total=427819008
   [junit4]   2> NOTE: All tests run in this JVM: [DocValuesMultiTest, ZkNodePropsTest, AddSchemaFieldsUpdateProcessorFactoryTest, ResponseLogComponentTest, BasicDistributedZk2Test, TestConfigSets, DistributedSuggestComponentTest, TestObjectReleaseTracker, HdfsSyncSliceTest, DirectSolrSpellCheckerTest, TestHdfsUpdateLog, TestFoldingMultitermQuery, TestRTGBase, OverseerTest, TimeZoneUtilsTest, DeleteInactiveReplicaTest, TestFuzzyAnalyzedSuggestions, TestHashQParserPlugin, TestReqParamsAPI, DocumentAnalysisRequestHandlerTest, TestHighlightDedupGrouping, TestInfoStreamLogging, ShardRoutingCustomTest, FieldMutatingUpdateProcessorTest, RegexBoostProcessorTest, StatsComponentTest, PreAnalyzedFieldTest, TestSurroundQueryParser, TestSolrQueryParser, TestOrdValues, RequestHandlersTest, DistribDocExpirationUpdateProcessorTest, TestExtendedDismaxParser, HdfsDirectoryFactoryTest, DeleteShardTest, DistributedQueryComponentCustomSortTest, RecoveryAfterSoftCommitTest, AnalysisAfterCoreReloadTest, RequiredFieldsTest, TestManagedStopFilterFactory, CSVRequestHandlerTest, RequestLoggingTest, TestInitParams, TestSweetSpotSimilarityFactory, LukeRequestHandlerTest, DistribJoinFromCollectionTest, TestRawResponseWriter, UUIDUpdateProcessorFallbackTest, TriLevelCompositeIdRoutingTest, TestIndexingPerformance, TestDynamicFieldResource, HardAutoCommitTest, TestSystemIdResolver, CloudExitableDirectoryReaderTest, TestSolrJ, SuggesterWFSTTest, TestStressReorder, CachingDirectoryFactoryTest, NumericFieldsTest, TestRemoteStreaming, TestCloudManagedSchemaConcurrent, TestReplicationHandler, CustomCollectionTest, ReplicationFactorTest]
   [junit4] Completed on J3 in 82.07s, 1 test, 1 error <<< FAILURES!

[...truncated 708 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 53 minutes 44 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 217369576 bytes
Compression is 0.0%
Took 43 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2898 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2898/

4 tests failed.
REGRESSION:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
There were too many update fails (28 > 20) - we expect it can happen, but shouldn't easily

Stack Trace:
java.lang.AssertionError: There were too many update fails (28 > 20) - we expect it can happen, but shouldn't easily
	at __randomizedtesting.SeedInfo.seed([D2B365EA0131FD74:5AE75A30AFCD908C]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertFalse(Assert.java:68)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:230)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:34624/d_k/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:34624/d_k/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([D2B365EA0131FD74:5AE75A30AFCD908C]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:34624 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:49592/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:49592/collection1
	at __randomizedtesting.SeedInfo.seed([D2B365EA0131FD74:5AE75A30AFCD908C]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:49592/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:49592 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22741/av_/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22741/av_/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([D2B365EA0131FD74:5AE75A30AFCD908C]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22741/av_/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:22741 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9377 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/init-core-data-001
   [junit4]   2> 102389 T454 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /av_/
   [junit4]   2> 102395 T454 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 102396 T455 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 102496 T454 oasc.ZkTestServer.run start zk server on port:22737
   [junit4]   2> 102497 T454 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 102498 T454 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 102501 T462 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@94552fd name:ZooKeeperConnection Watcher:127.0.0.1:22737 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 102501 T454 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 102502 T454 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 102502 T454 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 102505 T454 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 102506 T454 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 102507 T465 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7930f0f9 name:ZooKeeperConnection Watcher:127.0.0.1:22737/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 102508 T454 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 102508 T454 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 102509 T454 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 102512 T454 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 102515 T454 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 102517 T454 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 102520 T454 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 102520 T454 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 102524 T454 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 102524 T454 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 102527 T454 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 102528 T454 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 102530 T454 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 102531 T454 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 102534 T454 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 102534 T454 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 102537 T454 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 102537 T454 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 102540 T454 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 102540 T454 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 102543 T454 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 102543 T454 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 102546 T454 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 102546 T454 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 102549 T454 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 102549 T454 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 102552 T454 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 102552 T454 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 103542 T454 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/cores/collection1
   [junit4]   2> 103545 T454 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 103548 T454 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:22742
   [junit4]   2> 103548 T454 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/av_, hostPort=22741, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/tempDir-001/control/data}
   [junit4]   2> 103549 T454 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 103549 T454 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/'
   [junit4]   2> 103580 T454 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/solr.xml
   [junit4]   2> 103617 T454 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/cores
   [junit4]   2> 103618 T454 oasc.CoreContainer.<init> New CoreContainer 386184238
   [junit4]   2> 103618 T454 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/]
   [junit4]   2> 103619 T454 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/lib
   [junit4]   2> 103619 T454 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/lib).
   [junit4]   2> 103629 T454 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 103629 T454 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 103630 T454 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 103630 T454 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 103630 T454 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 103630 T454 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 103631 T454 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 103631 T454 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 103631 T454 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 103632 T454 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 103632 T454 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 103637 T454 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 103638 T454 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 103638 T454 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 103638 T454 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 103639 T454 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:22737/solr
   [junit4]   2> 103639 T454 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 103639 T454 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 103640 T454 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 103642 T477 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3e3a021a name:ZooKeeperConnection Watcher:127.0.0.1:22737 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 103642 T454 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 103643 T454 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 103645 T454 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 103646 T480 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5c20d4db name:ZooKeeperConnection Watcher:127.0.0.1:22737/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 103647 T454 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 103648 T454 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 103652 T454 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 103655 T454 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 103658 T454 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 103661 T454 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 103664 T454 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 103667 T454 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 103668 T454 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 103670 T454 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:22741_av_
   [junit4]   2> 103671 T454 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:22741_av_
   [junit4]   2> 103674 T454 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 103676 T454 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 103678 T454 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 103680 T454 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:22741_av_
   [junit4]   2> 103680 T454 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 103683 T454 oasc.Overseer.start Overseer (id=93605607187087363-127.0.0.1:22741_av_-n_0000000000) starting
   [junit4]   2> 103686 T454 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 103695 T454 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 103696 T482 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 103696 T454 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 103697 T481 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 103703 T454 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/cores
   [junit4]   2> 103712 T454 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, loadOnStartup=true, collection=control_collection, schema=schema.xml, coreNodeName=, name=collection1, shard=, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/cores/collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/cores/collection1/, dataDir=data/}
   [junit4]   2> 103712 T454 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/cores/collection1/
   [junit4]   2> 103712 T454 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 103714 T484 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 103714 T484 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 103719 T480 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 103719 T484 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 103724 T481 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22741/av_",
   [junit4]   2> 	  "node_name":"127.0.0.1:22741_av_"} current state version: 0
   [junit4]   2> 103756 T481 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22741/av_",
   [junit4]   2> 	  "node_name":"127.0.0.1:22741_av_"}
   [junit4]   2> 103760 T481 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 103794 T481 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 103806 T480 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 104721 T484 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 104721 T484 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 104722 T484 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 104722 T484 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 104724 T484 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 104724 T484 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/cores/collection1/'
   [junit4]   2> 104758 T484 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 104775 T484 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 104799 T484 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 104823 T484 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 104862 T484 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 104864 T484 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 104886 T484 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 105413 T484 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 105453 T484 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 105457 T484 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 105509 T484 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 105518 T484 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 105526 T484 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 105528 T484 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 105529 T484 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 105530 T484 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 105532 T484 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 105533 T484 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 105534 T484 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 105534 T484 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 105535 T484 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/cores/collection1/data/
   [junit4]   2> 105536 T484 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2baec60c
   [junit4]   2> 105564 T484 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/cores/collection1/data
   [junit4]   2> 105565 T484 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/cores/collection1/data/index/
   [junit4]   2> 105565 T484 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 105566 T484 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/cores/collection1/data/index
   [junit4]   2> 105567 T484 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=23, maxMergeAtOnceExplicit=46, maxMergedSegmentMB=19.234375, floorSegmentMB=1.67578125, forceMergeDeletesPctAllowed=20.4478361701504, segmentsPerTier=45.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 105568 T484 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6ce08c10 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@a0ba648),segFN=segments_1,generation=1}
   [junit4]   2> 105568 T484 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 105581 T484 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 105582 T484 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 105583 T484 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 105583 T484 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 105584 T484 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 105584 T484 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 105585 T484 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 105586 T484 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 105587 T484 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 105627 T484 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 105643 T484 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 105649 T484 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 105654 T484 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 105672 T484 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/ping,/update/json/docs,/admin/mbeans,/admin/logging,/admin/luke,/update/csv,/admin/system,/get,/admin/file,/admin/segments,/admin/properties,/schema,/update,/config,standard,/admin/threads,/admin/plugins,/replication,/update/json
   [junit4]   2> 105675 T484 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 105677 T484 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 105678 T484 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 105679 T484 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 105680 T484 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 105682 T484 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1228413357, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 105683 T484 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6ce08c10 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@a0ba648),segFN=segments_1,generation=1}
   [junit4]   2> 105684 T484 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 105684 T484 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@5de98963[collection1] main
   [junit4]   2> 105685 T484 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 105687 T484 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 105687 T484 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 105699 T484 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 105699 T484 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 105700 T484 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 105701 T484 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 105701 T484 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 105702 T484 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 105702 T484 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 105705 T485 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5de98963[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 105710 T484 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 105711 T488 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:22741/av_ collection:control_collection shard:shard1
   [junit4]   2> 105712 T454 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 105712 T454 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 105731 T488 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 105733 T454 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 105734 T454 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 105736 T492 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@202e1df1 name:ZooKeeperConnection Watcher:127.0.0.1:22737/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 105736 T454 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 105737 T454 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 105738 T454 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 105739 T488 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 105741 T480 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 105742 T488 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 105743 T488 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C10 name=collection1 org.apache.solr.core.SolrCore@74b4dce5 url=http://127.0.0.1:22741/av_/collection1 node=127.0.0.1:22741_av_ C10_STATE=coll:control_collection core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:22741/av_, node_name=127.0.0.1:22741_av_}
   [junit4]   2> 105743 T488 C:control_collection S:shard1 C10 P22741 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:22741/av_/collection1/
   [junit4]   2> 105743 T481 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 105743 T488 C:control_collection S:shard1 C10 P22741 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 105744 T488 C:control_collection S:shard1 C10 P22741 oasc.SyncStrategy.syncToMe http://127.0.0.1:22741/av_/collection1/ has no replicas
   [junit4]   2> 105744 T488 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:22741/av_/collection1/ shard1
   [junit4]   2> 105745 T488 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 105747 T454 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 105754 T492 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 105754 T480 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 105760 T481 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22741/av_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 105763 T480 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 105765 T480 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 105765 T492 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 105801 T488 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:22741/av_/collection1/ and leader is http://127.0.0.1:22741/av_/collection1/
   [junit4]   2> 105802 T488 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:22741/av_
   [junit4]   2> 105802 T488 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 105802 T488 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 105803 T488 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 105804 T480 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 105806 T481 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22741/av_",
   [junit4]   2> 	  "node_name":"127.0.0.1:22741_av_"} current state version: 3
   [junit4]   2> 105807 T481 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22741/av_",
   [junit4]   2> 	  "node_name":"127.0.0.1:22741_av_"}
   [junit4]   2> 105912 T480 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 105912 T492 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 107259 T454 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/cores/collection1
   [junit4]   2> 107261 T454 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001
   [junit4]   2> 107262 T454 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 107264 T454 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:22764
   [junit4]   2> 107265 T454 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=22763, solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/tempDir-001/jetty1, hostContext=/av_}
   [junit4]   2> 107265 T454 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 107266 T454 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/'
   [junit4]   2> 107302 T454 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/solr.xml
   [junit4]   2> 107343 T454 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/cores
   [junit4]   2> 107344 T454 oasc.CoreContainer.<init> New CoreContainer 807977474
   [junit4]   2> 107344 T454 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/]
   [junit4]   2> 107345 T454 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/lib
   [junit4]   2> 107345 T454 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/lib).
   [junit4]   2> 107355 T454 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 107355 T454 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 107356 T454 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 107356 T454 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 107356 T454 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 107357 T454 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 107357 T454 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 107357 T454 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 107358 T454 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 107358 T454 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 107358 T454 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 107363 T454 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 107364 T454 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 107365 T454 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 107365 T454 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 107365 T454 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:22737/solr
   [junit4]   2> 107366 T454 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 107366 T454 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 107367 T454 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 107369 T504 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5e7839da name:ZooKeeperConnection Watcher:127.0.0.1:22737 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 107370 T454 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 107370 T454 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 107373 T454 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 107374 T507 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@12dca9fc name:ZooKeeperConnection Watcher:127.0.0.1:22737/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 107375 T454 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 107382 T454 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 108391 T454 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:22763_av_
   [junit4]   2> 108392 T454 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:22763_av_
   [junit4]   2> 108397 T454 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 108400 T454 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/cores
   [junit4]   2> 108402 T454 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {loadOnStartup=true, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/cores/collection1/, coreNodeName=, collection=collection1, shard=, dataDir=data/, transient=false, config=solrconfig.xml, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/cores/collection1, name=collection1}
   [junit4]   2> 108403 T454 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/cores/collection1/
   [junit4]   2> 108403 T454 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 108404 T508 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 108405 T508 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 108406 T480 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 108406 T508 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 108408 T481 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22763/av_",
   [junit4]   2> 	  "node_name":"127.0.0.1:22763_av_"} current state version: 4
   [junit4]   2> 108409 T481 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22763/av_",
   [junit4]   2> 	  "node_name":"127.0.0.1:22763_av_"}
   [junit4]   2> 108409 T481 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 108409 T481 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 108411 T507 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 108411 T480 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 108411 T492 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 109407 T508 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 109408 T508 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 109409 T508 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 109410 T508 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 109411 T508 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 109411 T508 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/cores/collection1/'
   [junit4]   2> 109431 T508 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 109434 T508 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 109454 T508 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 109482 T508 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 109544 T508 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 109546 T508 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 109578 T508 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 110017 T508 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 110036 T508 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 110038 T508 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 110060 T508 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 110065 T508 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 110070 T508 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 110071 T508 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 110071 T508 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 110072 T508 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 110073 T508 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 110073 T508 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 110073 T508 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 110074 T508 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 110074 T508 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 110075 T508 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2baec60c
   [junit4]   2> 110076 T508 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/cores/collection1/data
   [junit4]   2> 110076 T508 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 110076 T508 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 110078 T508 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 110079 T508 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=23, maxMergeAtOnceExplicit=46, maxMergedSegmentMB=19.234375, floorSegmentMB=1.67578125, forceMergeDeletesPctAllowed=20.4478361701504, segmentsPerTier=45.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 110079 T508 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@63620338 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@37c18481),segFN=segments_1,generation=1}
   [junit4]   2> 110080 T508 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 110134 T508 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 110135 T508 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 110136 T508 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 110136 T508 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 110136 T508 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 110137 T508 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 110137 T508 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 110137 T508 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 110138 T508 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 110151 T508 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 110155 T508 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 110158 T508 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 110160 T508 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 110188 T508 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/ping,/update/json/docs,/admin/mbeans,/admin/logging,/admin/luke,/update/csv,/admin/system,/get,/admin/file,/admin/segments,/admin/properties,/schema,/update,/config,standard,/admin/threads,/admin/plugins,/replication,/update/json
   [junit4]   2> 110190 T508 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 110191 T508 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 110191 T508 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 110193 T508 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 110193 T508 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 110195 T508 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1228413357, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 110196 T508 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@63620338 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@37c18481),segFN=segments_1,generation=1}
   [junit4]   2> 110196 T508 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 110197 T508 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@7f774441[collection1] main
   [junit4]   2> 110197 T508 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 110199 T508 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 110199 T508 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 110200 T508 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 110200 T508 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 110200 T508 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 110201 T508 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 110202 T508 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 110202 T508 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 110202 T508 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 110206 T509 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7f774441[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 110207 T508 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 110208 T512 C:collection1 S:shard3 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:22763/av_ collection:collection1 shard:shard3
   [junit4]   2> 110208 T454 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 110209 T454 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 110209 T512 C:collection1 S:shard3 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
   [junit4]   2> 110215 T512 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard3
   [junit4]   2> 110217 T480 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 110217 T512 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 110218 T512 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C11 name=collection1 org.apache.solr.core.SolrCore@49daad2b url=http://127.0.0.1:22763/av_/collection1 node=127.0.0.1:22763_av_ C11_STATE=coll:collection1 core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:22763/av_, node_name=127.0.0.1:22763_av_}
   [junit4]   2> 110218 T512 C:collection1 S:shard3 C11 P22763 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:22763/av_/collection1/
   [junit4]   2> 110218 T481 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 110218 T512 C:collection1 S:shard3 C11 P22763 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 110219 T512 C:collection1 S:shard3 C11 P22763 oasc.SyncStrategy.syncToMe http://127.0.0.1:22763/av_/collection1/ has no replicas
   [junit4]   2> 110219 T512 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:22763/av_/collection1/ shard3
   [junit4]   2> 110220 T512 C:collection1 S:shard3 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard3
   [junit4]   2> 110221 T507 oasc

[...truncated too long message...]

e/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1598541 T5214 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:59868/
   [junit4]   2> 1598541 T5214 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 6 connections to: http://127.0.0.1:49592/
   [junit4]   2> 1598542 T5214 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:59214/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=D2B365EA0131FD74 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=sv_SE -Dtests.timezone=America/Bogota -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   71.2s J3 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:49592/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([D2B365EA0131FD74:5AE75A30AFCD908C]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:49592/collection1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:49592 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 54 more
   [junit4]   2> 1598568 T5214 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest D2B365EA0131FD74-001
   [junit4]   2> 71227 T5213 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {id=PostingsFormat(name=MockRandom), a_t=PostingsFormat(name=LuceneVarGapFixedInterval), text=Lucene50(blocksize=128), timestamp=PostingsFormat(name=LuceneVarGapFixedInterval), range_facet_l=PostingsFormat(name=LuceneVarGapFixedInterval), multiDefault=PostingsFormat(name=Memory doPackFST= false), intDefault=PostingsFormat(name=LuceneVarGapFixedInterval), _version_=PostingsFormat(name=LuceneVarGapFixedInterval)}, docValues:{timestamp=DocValuesFormat(name=Memory)}, sim=DefaultSimilarity, locale=sv_SE, timezone=America/Bogota
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=170346632,total=387973120
   [junit4]   2> NOTE: All tests run in this JVM: [SignatureUpdateProcessorFactoryTest, UnloadDistributedZkTest, StressHdfsTest, ShowFileRequestHandlerTest, ResponseLogComponentTest, TestChildDocTransformer, TestSystemIdResolver, SpatialRPTFieldTypeTest, TestLRUStatsCache, IndexBasedSpellCheckerTest, BinaryUpdateRequestHandlerTest, SpatialHeatmapFacetsTest, TestZkChroot, SpellPossibilityIteratorTest, TestComplexPhraseQParserPlugin, DistributedQueryComponentOptimizationTest, ExternalFileFieldSortTest, TestShardHandlerFactory, TestRandomMergePolicy, AutoCommitTest, SpellCheckComponentTest, TestLMDirichletSimilarityFactory, TestCodecSupport, OverseerRolesTest, TestJmxMonitoredMap, TestInitParams, TestPhraseSuggestions, SegmentsInfoRequestHandlerTest, TestReload, TestFileDictionaryLookup, TestQueryUtils, TestReversedWildcardFilterFactory, CursorPagingTest, AssignTest, TestMissingGroups, TestFaceting, CoreAdminHandlerTest, TestTrie, HighlighterTest, TestFieldTypeCollectionResource, TestCSVResponseWriter, CurrencyFieldXmlFileTest, SSLMigrationTest, TestSimpleQParserPlugin, DefaultValueUpdateProcessorTest, TestObjectReleaseTracker, TestReplicaProperties, EnumFieldTest, TestSolrDynamicMBean, BasicDistributedZk2Test, TestRequestStatusCollectionAPI, TimeZoneUtilsTest, PolyFieldTest, TestManagedSynonymFilterFactory, DistributedSuggestComponentTest, DistributedQueueTest, SuggesterWFSTTest, TestComponentsName, TestCloudManagedSchemaConcurrent, TestDistributedSearch, CollectionsAPIDistributedZkTest, TestCloudManagedSchema, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J3 in 71.50s, 1 test, 1 error <<< FAILURES!

[...truncated 691 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 1 failure, 61 ignored (21 assumptions)

Total time: 55 minutes 6 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 219846757 bytes
Compression is 0.0%
Took 42 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2897 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2897/

5 tests failed.
REGRESSION:  org.apache.solr.handler.TestReqParamsAPI.test

Error Message:
Could not get expected value  'fomrequest' for path 'params/a' full output: null

Stack Trace:
java.lang.AssertionError: Could not get expected value  'fomrequest' for path 'params/a' full output: null
	at __randomizedtesting.SeedInfo.seed([4DEA682486127912:C5BE57FE28EE14EA]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.core.TestSolrConfigHandler.testForResponseElement(TestSolrConfigHandler.java:405)
	at org.apache.solr.handler.TestReqParamsAPI.testReqParams(TestReqParamsAPI.java:125)
	at org.apache.solr.handler.TestReqParamsAPI.test(TestReqParamsAPI.java:71)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.solr.handler.component.DistributedMLTComponentTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:60625/wip/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:60625/wip/collection1
	at __randomizedtesting.SeedInfo.seed([4DEA682486127912:C5BE57FE28EE14EA]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:566)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:558)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:606)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:588)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:567)
	at org.apache.solr.handler.component.DistributedMLTComponentTest.test(DistributedMLTComponentTest.java:126)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 51 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:19178/ju/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19178/ju/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([4DEA682486127912:C5BE57FE28EE14EA]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:19178 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:30763/_b/u/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:30763/_b/u/collection1
	at __randomizedtesting.SeedInfo.seed([4DEA682486127912:C5BE57FE28EE14EA]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:30763/_b/u/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:30763 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19236/zeii/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19236/zeii/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([4DEA682486127912:C5BE57FE28EE14EA]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19236/zeii/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:19236 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9364 lines...]
   [junit4] Suite: org.apache.solr.handler.TestReqParamsAPI
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/init-core-data-001
   [junit4]   2> 136455 T197 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 136455 T197 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /rbe/
   [junit4]   2> 136464 T197 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 136465 T198 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 136564 T197 oasc.ZkTestServer.run start zk server on port:31034
   [junit4]   2> 136565 T197 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 136567 T197 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 136570 T205 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3855f423 name:ZooKeeperConnection Watcher:127.0.0.1:31034 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 136571 T197 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 136572 T197 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 136572 T197 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 136575 T197 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 136576 T197 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 136581 T208 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@70dcbc89 name:ZooKeeperConnection Watcher:127.0.0.1:31034/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 136581 T197 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 136582 T197 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 136582 T197 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 136585 T197 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 136587 T197 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 136588 T197 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 136590 T197 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 136591 T197 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 136594 T197 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 136595 T197 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 136700 T197 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 136702 T197 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 136707 T197 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 136708 T197 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 136711 T197 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 136712 T197 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 136716 T197 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 136718 T197 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 136723 T197 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 136724 T197 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 136729 T197 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 136731 T197 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 136736 T197 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 136738 T197 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 136742 T197 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 136744 T197 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 136749 T197 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 136751 T197 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 140251 T197 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/cores/collection1
   [junit4]   2> 140262 T197 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 140270 T197 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:55955
   [junit4]   2> 140270 T197 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/cores, hostPort=55955, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/tempDir-001/control/data, hostContext=/rbe}
   [junit4]   2> 140271 T197 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@749cd006
   [junit4]   2> 140272 T197 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/'
   [junit4]   2> 140421 T197 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/solr.xml
   [junit4]   2> 140625 T197 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/cores
   [junit4]   2> 140626 T197 oasc.CoreContainer.<init> New CoreContainer 514480383
   [junit4]   2> 140626 T197 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/]
   [junit4]   2> 140627 T197 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/lib
   [junit4]   2> 140631 T197 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/lib).
   [junit4]   2> 140650 T197 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 140651 T197 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 140652 T197 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 140652 T197 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 140653 T197 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 140653 T197 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 140653 T197 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 140654 T197 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 140655 T197 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 140655 T197 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 140656 T197 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 140683 T197 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 140686 T197 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 140686 T197 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 140687 T197 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 140687 T197 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:31034/solr
   [junit4]   2> 140688 T197 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 140689 T197 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 140691 T197 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 140695 T220 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3e8aef88 name:ZooKeeperConnection Watcher:127.0.0.1:31034 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 140696 T197 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 140698 T197 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 140702 T197 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 140705 T223 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@582e177e name:ZooKeeperConnection Watcher:127.0.0.1:31034/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 140705 T197 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 140708 T197 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 140711 T197 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 140714 T197 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 140716 T197 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 140719 T197 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 140723 T197 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 140726 T197 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 140728 T197 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 140734 T197 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55955_rbe
   [junit4]   2> 140736 T197 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:55955_rbe
   [junit4]   2> 140738 T197 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 140741 T197 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 140744 T197 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 140748 T197 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:55955_rbe
   [junit4]   2> 140749 T197 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 140750 T197 oasc.Overseer.start Overseer (id=93604667402813443-127.0.0.1:55955_rbe-n_0000000000) starting
   [junit4]   2> 140754 T197 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 140773 T197 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 140774 T225 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 140774 T197 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 140775 T224 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 140784 T197 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/cores
   [junit4]   2> 140799 T197 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, name=collection1, loadOnStartup=true, config=solrconfig.xml, collection=control_collection, dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/cores/collection1/, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/cores/collection1, transient=false, coreNodeName=}
   [junit4]   2> 140800 T197 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/cores/collection1/
   [junit4]   2> 140801 T197 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 140803 T227 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 140804 T227 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 140806 T223 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 140806 T227 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 140809 T224 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:55955_rbe",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:55955/rbe",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null} current state version: 0
   [junit4]   2> 140810 T224 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:55955_rbe",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:55955/rbe",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 140811 T224 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 140812 T224 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 140814 T223 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 141808 T227 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 141809 T227 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 141811 T227 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 141812 T227 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 141814 T227 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 141815 T227 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/cores/collection1/'
   [junit4]   2> 141846 T227 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 141853 T227 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 141872 T227 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 141937 T227 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 142040 T227 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 142142 T227 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 142215 T227 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 142962 T227 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 143009 T227 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 143015 T227 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 143057 T227 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 143071 T227 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 143082 T227 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 143085 T227 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 143086 T227 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 143086 T227 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 143088 T227 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 143089 T227 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 143089 T227 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 143090 T227 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 143091 T227 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/cores/collection1/data/
   [junit4]   2> 143091 T227 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@545319cc
   [junit4]   2> 143093 T227 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/cores/collection1/data
   [junit4]   2> 143093 T227 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/cores/collection1/data/index/
   [junit4]   2> 143094 T227 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 143094 T227 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/cores/collection1/data/index
   [junit4]   2> 143095 T227 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=49, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.20099856643417963]
   [junit4]   2> 143097 T227 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@750eb0fb lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@17856ac7),segFN=segments_1,generation=1}
   [junit4]   2> 143098 T227 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 143108 T227 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 143109 T227 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 143110 T227 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 143111 T227 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 143111 T227 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 143112 T227 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 143113 T227 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 143113 T227 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 143114 T227 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 143187 T227 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 143192 T227 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 143198 T227 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 143206 T227 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 143217 T227 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/ping,/admin/threads,/update/json,/admin/luke,/admin/file,/replication,/admin/properties,/update,/admin/mbeans,standard,/schema,/admin/logging,/admin/plugins,/admin/system,/get,/update/csv,/config,/update/json/docs,/admin/segments
   [junit4]   2> 143222 T227 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 143227 T227 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 143228 T227 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 143235 T227 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 143236 T227 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 143238 T227 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=38, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 143239 T227 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@750eb0fb lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@17856ac7),segFN=segments_1,generation=1}
   [junit4]   2> 143240 T227 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 143240 T227 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@1faeb8[collection1] main
   [junit4]   2> 143241 T227 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 143243 T227 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 143243 T227 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 143244 T227 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 143244 T227 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 143245 T227 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 143246 T227 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 143247 T227 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 143247 T227 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 143247 T227 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 143249 T228 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1faeb8[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 143249 T227 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 143250 T231 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:55955/rbe collection:control_collection shard:shard1
   [junit4]   2> 143251 T197 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 143251 T197 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 143253 T231 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 143255 T197 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 143256 T197 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 143258 T234 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@544187a0 name:ZooKeeperConnection Watcher:127.0.0.1:31034/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 143259 T197 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 143259 T231 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 143260 T197 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 143260 T197 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 143263 T223 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 143264 T231 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 143264 T231 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C13 name=collection1 org.apache.solr.core.SolrCore@11f56172 url=http://127.0.0.1:55955/rbe/collection1 node=127.0.0.1:55955_rbe C13_STATE=coll:control_collection core:collection1 props:{state=down, node_name=127.0.0.1:55955_rbe, core=collection1, base_url=http://127.0.0.1:55955/rbe}
   [junit4]   2> 143265 T231 C:control_collection S:shard1 C13 P55955 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:55955/rbe/collection1/
   [junit4]   2> 143265 T224 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 143266 T231 C:control_collection S:shard1 C13 P55955 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 143265 T197 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 143267 T231 C:control_collection S:shard1 C13 P55955 oasc.SyncStrategy.syncToMe http://127.0.0.1:55955/rbe/collection1/ has no replicas
   [junit4]   2> 143267 T197 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 143269 T231 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:55955/rbe/collection1/ shard1
   [junit4]   2> 143270 T223 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 143270 T234 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 143270 T197 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 143270 T231 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 143273 T197 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 143275 T237 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4b725081 name:ZooKeeperConnection Watcher:127.0.0.1:31034/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 143276 T197 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 143277 T197 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 143280 T223 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 143282 T224 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 143283 T224 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 143287 T224 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:55955/rbe",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 143289 T224 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 143291 T234 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 143291 T223 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 143297 T223 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 143298 T234 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 143298 T223 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 143335 T231 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:55955/rbe/collection1/ and leader is http://127.0.0.1:55955/rbe/collection1/
   [junit4]   2> 143335 T231 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:55955/rbe
   [junit4]   2> 143336 T231 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 143336 T231 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 143336 T231 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 143339 T223 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 143342 T224 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:55955_rbe",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:55955/rbe",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "roles":null} current state version: 4
   [junit4]   2> 143343 T224 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:55955_rbe",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:55955/rbe",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 143448 T223 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 143448 T234 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 146481 T197 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/shard-1-001/cores/collection1
   [junit4]   2> 146505 T197 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/shard-1-001
   [junit4]   2> 146506 T197 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 146513 T197 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:35051
   [junit4]   2> 146516 T197 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/rbe, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/shard-1-001/cores, hostPort=35051}
   [junit4]   2> 146518 T197 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@749cd006
   [junit4]   2> 146519 T197 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/shard-1-001/'
   [junit4]   2> 146671 T197 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/shard-1-001/solr.xml
   [junit4]   2> 146802 T197 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/shard-1-001/cores
   [junit4]   2> 146804 T197 oasc.CoreContainer.<init> New CoreContainer 1594767538
   [junit4]   2> 146804 T197 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/shard-1-001/]
   [junit4]   2> 146805 T197 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/shard-1-001/lib
   [junit4]   2> 146808 T197 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/shard-1-001/lib).
   [junit4]   2> 146834 T197 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 146835 T197 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 146836 T197 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 146836 T197 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 146837 T197 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 146838 T197 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 146839 T197 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 146839 T197 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 146840 T197 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 146840 T197 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 146841 T197 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 146861 T197 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 146863 T197 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 146864 T197 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 146864 T197 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 146865 T197 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:31034/solr
   [junit4]   2> 146865 T197 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 146866 T197 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 146867 T197 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 146869 T249 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@44d916e6 name:ZooKeeperConnection Watcher:127.0.0.1:31034 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 146870 T197 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 146871 T197 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 146875 T197 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 146877 T252 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@22fb957a name:ZooKeeperConnection Watcher:127.0.0.1:31034/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 146878 T197 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 146887 T197 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 147900 T197 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35051_rbe
   [junit4]   2> 147902 T197 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:35051_rbe
   [junit4]   2> 147909 T197 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 147915 T197 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/shard-1-001/cores
   [junit4]   2> 147926 T197 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, loadOnStartup=true, dataDir=data/, name=collection1, collection=collection1, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/shard-1-001/cores/collection1, shard=, schema=schema.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/shard-1-001/cores/collection1/, transient=false}
   [junit4]   2> 147927 T197 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/shard-1-001/cores/collection1/
   [junit4]   2> 147928 T197 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 147935 T253 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 147936 T253 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 147939 T223 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 147942 T224 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:35051_rbe",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:35051/rbe",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "roles":null} current state version: 5
   [junit4]   2> 147943 T224 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:35051_rbe",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:35051/rbe",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 147943 T224 oasco.ReplicaMutator.updateState Collection already exists with numShards=2
   [junit4]   2> 147944 T224 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 147947 T224 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 147959 T253 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 147959 T253 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 147960 T253 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 147962 T253 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 147962 T253 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 147962 T253 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 147963 T253 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 147964 T253 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 147964 T253 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 147968 T253 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 147969 T253 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/shard-1-001/cores/collection1/'
   [junit4]   2> 148002 T253 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 148011 T253 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 148036 T253 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 148092 T253 C:collection1 c:collect

[...truncated too long message...]

ol_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 11 connections to: http://127.0.0.1:19236/zeii
   [junit4]   2> 1745441 T3579 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:19261/zeii
   [junit4]   2> 1745441 T3579 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 1 connections to: http://127.0.0.1:19215/zeii
   [junit4]   2> 1745441 T3579 C:control_collection S:shard1 c:collection1 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1745442 T3579 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:19293/zeii
   [junit4]   2> 1745442 T3579 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:19236/zeii
   [junit4]   2> 1745442 T3579 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:19261/zeii
   [junit4]   2> 1745442 T3579 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:19215/zeii
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=4DEA682486127912 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=cs -Dtests.timezone=America/Kralendijk -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   79.8s J0 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19236/zeii/repfacttest_c8n_1x3_shard1_replica3
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([4DEA682486127912:C5BE57FE28EE14EA]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19236/zeii/repfacttest_c8n_1x3_shard1_replica3
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:19236 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 53 more
   [junit4]   2> 1745463 T3579 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4DEA682486127912-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {intDefault=Lucene50(blocksize=128), range_facet_l=Lucene50(blocksize=128), id=PostingsFormat(name=Memory doPackFST= false), text=FSTOrd50, a_t=Lucene50(blocksize=128), multiDefault=PostingsFormat(name=LuceneVarGapFixedInterval), timestamp=Lucene50(blocksize=128), _version_=Lucene50(blocksize=128)}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=DefaultSimilarity, locale=cs, timezone=America/Kralendijk
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=185321288,total=279445504
   [junit4]   2> NOTE: All tests run in this JVM: [PreAnalyzedUpdateProcessorTest, TestBlobHandler, HdfsChaosMonkeySafeLeaderTest, TestManagedSchemaFieldTypeResource, TestReqParamsAPI, DistributedMLTComponentTest, SpatialFilterTest, InfoHandlerTest, DOMUtilTest, TestSortingResponseWriter, TestAnalyzeInfixSuggestions, HdfsRecoveryZkTest, TestConfigSets, TestOverriddenPrefixQueryForCustomFieldType, CoreAdminHandlerTest, TestSerializedLuceneMatchVersion, DistributedSpellCheckComponentTest, HighlighterMaxOffsetTest, DeleteLastCustomShardedReplicaTest, TestStressLucene, RegexBoostProcessorTest, HttpPartitionTest, TestPseudoReturnFields, BadIndexSchemaTest, SpellPossibilityIteratorTest, TestFastLRUCache, TestStressVersions, TestSort, TestLMJelinekMercerSimilarityFactory, SuggesterTSTTest, LeaderElectionTest, DistanceFunctionTest, DocValuesTest, MultiTermTest, DistribJoinFromCollectionTest, TestFieldCollectionResource, FileUtilsTest, OverseerStatusTest, TestLuceneMatchVersion, TestNamedUpdateProcessors, MultiThreadedOCPTest, AutoCommitTest, TestIntervalFaceting, DocumentAnalysisRequestHandlerTest, BasicZkTest, TestDocumentBuilder, TestManagedResource, CurrencyFieldOpenExchangeTest, TestGroupingSearch, DistributedTermsComponentTest, AnalysisErrorHandlingTest, TestReRankQParserPlugin, TestFuzzyAnalyzedSuggestions, TestSchemaSimilarityResource, BadCopyFieldTest, DefaultValueUpdateProcessorTest, TestIndexSearcher, DirectSolrSpellCheckerTest, ReplicationFactorTest]
   [junit4] Completed on J0 in 79.83s, 1 test, 1 error <<< FAILURES!

[...truncated 711 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 4 errors, 1 failure, 61 ignored (21 assumptions)

Total time: 58 minutes 54 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 222722426 bytes
Compression is 0.0%
Took 42 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2896 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2896/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:51146/ad_lr/q/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:51146/ad_lr/q/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([AB409680AE49383E:2314A95A00B555C6]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:51146 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28748/velr/s/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28748/velr/s/collection1
	at __randomizedtesting.SeedInfo.seed([AB409680AE49383E:2314A95A00B555C6]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28748/velr/s/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:28748 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22940/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22940/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([AB409680AE49383E:2314A95A00B555C6]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22940/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:22940 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9953 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/init-core-data-001
   [junit4]   2> 671811 T2102 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 671816 T2102 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 671816 T2103 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 671916 T2102 oasc.ZkTestServer.run start zk server on port:22936
   [junit4]   2> 671917 T2102 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 671917 T2102 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 671921 T2110 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@29d42291 name:ZooKeeperConnection Watcher:127.0.0.1:22936 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 671921 T2102 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 671922 T2102 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 671922 T2102 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 671924 T2102 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 671925 T2102 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 671926 T2113 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6487d71e name:ZooKeeperConnection Watcher:127.0.0.1:22936/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 671926 T2102 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 671927 T2102 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 671927 T2102 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 671928 T2102 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 671929 T2102 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 671930 T2102 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 671931 T2102 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 671932 T2102 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 671934 T2102 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 671934 T2102 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 672035 T2102 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 672036 T2102 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 672037 T2102 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 672037 T2102 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 672039 T2102 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 672039 T2102 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 672040 T2102 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 672041 T2102 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 672042 T2102 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 672042 T2102 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 672044 T2102 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 672044 T2102 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 672045 T2102 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 672046 T2102 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 672047 T2102 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 672048 T2102 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 672049 T2102 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 672049 T2102 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 673198 T2102 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/cores/collection1
   [junit4]   2> 673201 T2102 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 673204 T2102 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:22941
   [junit4]   2> 673204 T2102 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=22940, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/cores, hostContext=/}
   [junit4]   2> 673205 T2102 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 673205 T2102 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/'
   [junit4]   2> 673237 T2102 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/solr.xml
   [junit4]   2> 673286 T2102 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/cores
   [junit4]   2> 673286 T2102 oasc.CoreContainer.<init> New CoreContainer 1987784992
   [junit4]   2> 673286 T2102 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/]
   [junit4]   2> 673286 T2102 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/lib
   [junit4]   2> 673287 T2102 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/lib).
   [junit4]   2> 673297 T2102 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 673297 T2102 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 673298 T2102 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 673298 T2102 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 673298 T2102 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 673298 T2102 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 673298 T2102 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 673299 T2102 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 673299 T2102 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 673299 T2102 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 673299 T2102 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 673305 T2102 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 673305 T2102 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 673306 T2102 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 673306 T2102 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 673306 T2102 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:22936/solr
   [junit4]   2> 673306 T2102 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 673306 T2102 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 673307 T2102 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 673309 T2125 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@56c7eda0 name:ZooKeeperConnection Watcher:127.0.0.1:22936 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 673309 T2102 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 673310 T2102 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 673311 T2102 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 673322 T2128 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7f78b60b name:ZooKeeperConnection Watcher:127.0.0.1:22936/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 673322 T2102 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 673323 T2102 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 673325 T2102 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 673326 T2102 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 673328 T2102 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 673329 T2102 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 673331 T2102 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 673332 T2102 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 673333 T2102 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 673333 T2102 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:22940_
   [junit4]   2> 673334 T2102 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:22940_
   [junit4]   2> 673335 T2102 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 673336 T2102 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 673337 T2102 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 673338 T2102 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:22940_
   [junit4]   2> 673338 T2102 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 673339 T2102 oasc.Overseer.start Overseer (id=93603851893276675-127.0.0.1:22940_-n_0000000000) starting
   [junit4]   2> 673341 T2102 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 673345 T2102 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 673345 T2130 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 673345 T2102 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 673346 T2129 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 673349 T2102 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/cores
   [junit4]   2> 673351 T2102 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, schema=schema.xml, coreNodeName=, collection=control_collection, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/cores/collection1, loadOnStartup=true, dataDir=data/, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/cores/collection1/, transient=false, name=collection1}
   [junit4]   2> 673351 T2102 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/cores/collection1/
   [junit4]   2> 673351 T2102 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 673352 T2132 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 673352 T2132 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 673353 T2132 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 673353 T2128 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 673353 T2129 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22940",
   [junit4]   2> 	  "node_name":"127.0.0.1:22940_"} current state version: 0
   [junit4]   2> 673354 T2129 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22940",
   [junit4]   2> 	  "node_name":"127.0.0.1:22940_"}
   [junit4]   2> 673354 T2129 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 673354 T2129 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 673355 T2128 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 674353 T2132 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 674354 T2132 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 674354 T2132 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 674354 T2132 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 674355 T2132 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 674355 T2132 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/cores/collection1/'
   [junit4]   2> 674368 T2132 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 674370 T2132 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 674379 T2132 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 674396 T2132 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 674434 T2132 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 674535 T2132 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 674542 T2132 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 674843 T2132 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 674855 T2132 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 674857 T2132 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 674865 T2132 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 674868 T2132 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 674870 T2132 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 674871 T2132 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 674872 T2132 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 674872 T2132 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 674873 T2132 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 674873 T2132 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 674873 T2132 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 674873 T2132 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 674873 T2132 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/cores/collection1/data/
   [junit4]   2> 674874 T2132 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@41ee863
   [junit4]   2> 674874 T2132 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/cores/collection1/data
   [junit4]   2> 674874 T2132 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/cores/collection1/data/index/
   [junit4]   2> 674875 T2132 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 674875 T2132 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/cores/collection1/data/index
   [junit4]   2> 674875 T2132 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 674876 T2132 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@bf1054e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7c934a01),segFN=segments_1,generation=1}
   [junit4]   2> 674876 T2132 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 674880 T2132 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 674880 T2132 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 674880 T2132 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 674880 T2132 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 674881 T2132 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 674881 T2132 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 674881 T2132 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 674881 T2132 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 674881 T2132 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 674897 T2132 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 674898 T2132 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 674900 T2132 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 674902 T2132 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 674905 T2132 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/file,/admin/mbeans,/admin/logging,/admin/ping,/update/json/docs,/config,/get,/admin/threads,/admin/luke,/schema,/admin/properties,/replication,/admin/segments,/update,/admin/system,/update/csv,/admin/plugins,standard,/update/json
   [junit4]   2> 674907 T2132 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 674908 T2132 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 674908 T2132 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 674909 T2132 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 674909 T2132 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 674910 T2132 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=37, maxMergeAtOnceExplicit=50, maxMergedSegmentMB=8.111328125, floorSegmentMB=1.8037109375, forceMergeDeletesPctAllowed=16.785093067273394, segmentsPerTier=24.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 674910 T2132 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@bf1054e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7c934a01),segFN=segments_1,generation=1}
   [junit4]   2> 674911 T2132 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 674911 T2132 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@7a95226[collection1] main
   [junit4]   2> 674911 T2132 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 674912 T2132 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 674912 T2132 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 674912 T2132 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 674913 T2132 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 674913 T2132 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 674913 T2132 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 674913 T2132 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 674913 T2132 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 674914 T2132 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 674914 T2132 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 674914 T2133 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7a95226[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 674915 T2136 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:22940 collection:control_collection shard:shard1
   [junit4]   2> 674915 T2102 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 674915 T2102 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 674915 T2136 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 674918 T2136 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 674919 T2102 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 674919 T2102 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 674920 T2128 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 674920 T2136 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 674920 T2136 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 674921 T2129 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C437 name=collection1 org.apache.solr.core.SolrCore@5cc2c07f url=http://127.0.0.1:22940/collection1 node=127.0.0.1:22940_ C437_STATE=coll:control_collection core:collection1 props:{state=down, core=collection1, base_url=http://127.0.0.1:22940, node_name=127.0.0.1:22940_}
   [junit4]   2> 674921 T2136 C:control_collection S:shard1 C437 P22940 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:22940/collection1/
   [junit4]   2> 674920 T2140 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@16e87b6a name:ZooKeeperConnection Watcher:127.0.0.1:22936/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 674921 T2136 C:control_collection S:shard1 C437 P22940 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 674921 T2102 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 674921 T2136 C:control_collection S:shard1 C437 P22940 oasc.SyncStrategy.syncToMe http://127.0.0.1:22940/collection1/ has no replicas
   [junit4]   2> 674922 T2102 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 674922 T2128 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 674922 T2102 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 674922 T2136 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:22940/collection1/ shard1
   [junit4]   2> 674922 T2136 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 674924 T2102 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 674924 T2102 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 674925 T2102 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 674925 T2102 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 674925 T2128 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 674926 T2143 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@94b4162 name:ZooKeeperConnection Watcher:127.0.0.1:22936/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 674926 T2129 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22940",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 674926 T2102 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 674927 T2102 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 674928 T2128 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 674928 T2129 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 674929 T2129 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 674929 T2128 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 674929 T2140 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 674976 T2136 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:22940/collection1/ and leader is http://127.0.0.1:22940/collection1/
   [junit4]   2> 674977 T2136 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:22940
   [junit4]   2> 674977 T2136 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 674977 T2136 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 674977 T2136 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 674978 T2128 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 674979 T2129 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22940",
   [junit4]   2> 	  "node_name":"127.0.0.1:22940_"} current state version: 3
   [junit4]   2> 674979 T2129 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22940",
   [junit4]   2> 	  "node_name":"127.0.0.1:22940_"}
   [junit4]   2> 674980 T2129 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 674981 T2140 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 674981 T2128 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 675088 T2140 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 675088 T2128 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 676496 T2102 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/cores/collection1
   [junit4]   2> 676500 T2102 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001
   [junit4]   2> 676501 T2102 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 676504 T2102 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:22948
   [junit4]   2> 676504 T2102 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostPort=22947, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/cores}
   [junit4]   2> 676505 T2102 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 676505 T2102 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/'
   [junit4]   2> 676560 T2102 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/solr.xml
   [junit4]   2> 676614 T2102 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/cores
   [junit4]   2> 676614 T2102 oasc.CoreContainer.<init> New CoreContainer 898692719
   [junit4]   2> 676615 T2102 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/]
   [junit4]   2> 676615 T2102 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/lib
   [junit4]   2> 676616 T2102 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/lib).
   [junit4]   2> 676626 T2102 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 676626 T2102 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 676626 T2102 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 676626 T2102 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 676626 T2102 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 676627 T2102 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 676627 T2102 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 676627 T2102 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 676627 T2102 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 676628 T2102 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 676628 T2102 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 676635 T2102 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 676636 T2102 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 676636 T2102 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 676636 T2102 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 676637 T2102 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:22936/solr
   [junit4]   2> 676637 T2102 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 676637 T2102 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 676638 T2102 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 676639 T2155 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6c1ffc19 name:ZooKeeperConnection Watcher:127.0.0.1:22936 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 676640 T2102 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 676640 T2102 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 676642 T2102 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 676643 T2158 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@132ee5ad name:ZooKeeperConnection Watcher:127.0.0.1:22936/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 676643 T2102 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 676645 T2102 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 677649 T2102 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:22947_
   [junit4]   2> 677650 T2102 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:22947_
   [junit4]   2> 677654 T2102 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 677656 T2102 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/cores
   [junit4]   2> 677659 T2102 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, dataDir=data/, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/cores/collection1, transient=false, schema=schema.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/cores/collection1/, collection=collection1, coreNodeName=, shard=, loadOnStartup=true}
   [junit4]   2> 677659 T2102 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/cores/collection1/
   [junit4]   2> 677659 T2102 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 677661 T2159 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 677661 T2159 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 677663 T2128 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 677663 T2159 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 677663 T2129 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22947",
   [junit4]   2> 	  "node_name":"127.0.0.1:22947_"} current state version: 5
   [junit4]   2> 677663 T2159 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 677664 T2129 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22947",
   [junit4]   2> 	  "node_name":"127.0.0.1:22947_"}
   [junit4]   2> 677664 T2129 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 677664 T2159 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 677664 T2129 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 677665 T2159 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 677666 T2129 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 677666 T2159 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 677666 T2158 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 677667 T2158 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 678666 T2159 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 678667 T2159 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 678667 T2159 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 678667 T2159 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 678668 T2159 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 678668 T2159 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/cores/collection1/'
   [junit4]   2> 678682 T2159 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 678684 T2159 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 678692 T2159 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 678708 T2159 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 678744 T2159 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 678845 T2159 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 678852 T2159 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 679130 T2159 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 679140 T2159 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 679142 T2159 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 679149 T2159 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 679160 T2159 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 679164 T2159 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 679164 T2159 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 679165 T2159 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 679165 T2159 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 679165 T2159 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 679165 T2159 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 679166 T2159 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 679166 T2159 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 679166 T2159 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 679166 T2159 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@41ee863
   [junit4]   2> 679167 T2159 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/cores/collection1/data
   [junit4]   2> 679167 T2159 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 679167 T2159 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 679168 T2159 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 679168 T2159 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 679169 T2159 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@793b06c8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@123a72a6),segFN=segments_1,generation=1}
   [junit4]   2> 679169 T2159 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 679173 T2159 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 679174 T2159 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 679174 T2159 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 679174 T2159 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 679174 T2159 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 679174 T2159 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 679175 T2159 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 679175 T2159 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 679175 T2159 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 679187 T2159 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 679189 T2159 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 679190 T2159 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 679192 T2159 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 679197 T2159 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/file,/admin/mbeans,/admin/logging,/admin/ping,/update/json/docs,/config,/get,/admin/threads,/admin/luke,/schema,/admin/properties,/replication,/admin/segments,/update,/admin/system,/update/csv,/admin/plugins,standard,/update/json
   [junit4]   2> 679199 T2159 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 679200 T2159 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 679200 T2159 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 679201 T2159 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 679201 T2159 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 679202 T2159 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=37, maxMergeAtOnceExplicit=50, maxMergedSegmentMB=8.111328125, floorSegmentMB=1.8037109375, forceMergeDeletesPctAllowed=16.785093067273394, segmentsPerTier=24.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 679202 T2159 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@793b06c8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@123a72a6),segFN=segments_1,generation=1}
   [junit4]   2> 679203 T2159 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 679203 T2159 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@67910daa[collection1] main
   [junit4]   2> 679203 T2159 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 679204 T2159 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 679204 T2159 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 679204 T2159 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 679204 T2159 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 679205 T2159 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 679205 T2159 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 679205 T2159 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 679205 T2159 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 679206 T2159 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 679207 T2160 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@67910daa[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 679207 T2159 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 679208 T2163 C:collection1 S:shard3 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:22947 collection:collection1 shard:shard3
   [junit4]   2> 679208 T2102 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 679208 T2102 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 679209 T2163 C:collection1 S:shard3 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
   [junit4]   2> 679212 T2163 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard3
   [junit4]   2> 679214 T2128 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 679214 T2163 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 679214 T2163 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C438 name=collection1 org.apache.solr.core.SolrCore@1f2ff3a2 url=http://127.0.0.1:22947/collection1 node=127.0.0.1:22947_ C438_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, base_url=http://127.0.0.1:22947, node_name=127.0.0.1:22947_}
   [junit4]   2> 679214 T2163 C:collection1 S:shard3 C438 P22947 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:22947/collection1/
   [junit4]   2> 679214 T2129 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 679215 T2163 C:collection1 S:shard3 C438 P22947 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 679215 T2163 C:collection1 S:shard3 C438 P22947 oasc.SyncStrategy.syncToMe http://127.0.0.1:22947/collection1/ has no replicas
   [junit4]   2> 679215 T2163 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:22947/collection1/ shard3
   [junit4]   2> 679215 T2163 C:collection1 S:shard3 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard3
   [junit4]   2> 679216 T2129 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 679216 T2158 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 679217 T2158 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 679219 T2129 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22947",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 679220 T2129 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 679220 T2158 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 679221 T2158 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 679222 T2128 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 679269 T2163 C:collection1 S:shard3 oasc.ZkController.register We are http://127.0.0.1:22947/collection1/ and leader is http://127.0.0.1:22947/collection1/
   [junit4]   2> 679270 T2163 C:collection1 S:shard3 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:22947
   [junit4]   2> 679270 T2163 C:collection1 S:shard3 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 679270 T2163 C:collection1 S:shard3 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 679270 T2163 C:collection1 S:shard3 c:collection1 oasc.ZkController.publish numSh

[...truncated too long message...]

andler{/velr/s,null}
   [junit4]   2> 1510573 T5402 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:62281 62281
   [junit4]   2> 1510607 T5403 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:62281 62281
   [junit4]   2> 1510610 T5403 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 		2	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1510610 T5402 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:34665/velr/s
   [junit4]   2> 1510611 T5402 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:28748/velr/s
   [junit4]   2> 1510611 T5402 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:21011/velr/s
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=AB409680AE49383E -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=no_NO -Dtests.timezone=Africa/Accra -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   67.8s J1 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28748/velr/s/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([AB409680AE49383E:2314A95A00B555C6]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28748/velr/s/collection1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:28748 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 54 more
   [junit4]   2> 1510639 T5402 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest AB409680AE49383E-001
   [junit4]   2> 67833 T5401 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50), sim=DefaultSimilarity, locale=no_NO, timezone=Africa/Accra
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=72927008,total=371195904
   [junit4]   2> NOTE: All tests run in this JVM: [PeerSyncTest, CloudMLTQParserTest, TestCollationField, OverseerRolesTest, TestSolrCoreProperties, AnalyticsQueryTest, TestFileDictionaryLookup, BlockDirectoryTest, SearchHandlerTest, ScriptEngineTest, TestRecovery, TestIBSimilarityFactory, TestIndexingPerformance, CursorMarkTest, SimpleMLTQParserTest, TestZkChroot, DefaultValueUpdateProcessorTest, TestComponentsName, SynonymTokenizerTest, OpenCloseCoreStressTest, DocumentAnalysisRequestHandlerTest, TestIntervalFaceting, TestCollectionAPI, CoreMergeIndexesAdminHandlerTest, CloudExitableDirectoryReaderTest, DocValuesTest, SharedFSAutoReplicaFailoverUtilsTest, TestPHPSerializedResponseWriter, TestRecoveryHdfs, AddBlockUpdateTest, DocumentBuilderTest, InfoHandlerTest, JsonLoaderTest, DistributedQueryComponentCustomSortTest, SliceStateTest, DistributedQueryElevationComponentTest, TestCSVLoader, ShardSplitTest, DistributedDebugComponentTest, TestManagedResourceStorage, SpellCheckComponentTest, CursorPagingTest, TestSearcherReuse, TestConfigSets, LeaderElectionTest, TestExceedMaxTermLength, TestShortCircuitedRequests, TestInfoStreamLogging, TestJettySolrRunner, TestSchemaManager, TestStandardQParsers, ResponseLogComponentTest, AnalysisAfterCoreReloadTest, TestAtomicUpdateErrorCases, ReplicationFactorTest, SpatialHeatmapFacetsTest, TestFieldTypeCollectionResource, TestStressUserVersions, TestCloudManagedSchemaConcurrent, TestDistributedSearch, MultiThreadedOCPTest, TestCloudManagedSchema, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J1 in 68.62s, 1 test, 1 error <<< FAILURES!

[...truncated 680 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 52 minutes 56 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 231796450 bytes
Compression is 0.0%
Took 44 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2895 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2895/

7 tests failed.
REGRESSION:  org.apache.solr.schema.TestCloudManagedSchemaConcurrent.test

Error Message:
QUERY FAILED: xpath=/response/lst[@name='responseHeader']/int[@name='status'][.='0']  request=/schema/copyfields/?wt=xml&updateTimeoutSecs=15  response=<?xml version="1.0" encoding="UTF-8"?> <response>  <lst name="responseHeader">   <int name="status">500</int>   <int name="QTime">46117</int> </lst> <lst name="error">   <str name="msg">7 out of 8 replicas failed to update their schema to version 25 within 15 seconds! Failed cores: [http://127.0.0.1:29575/ocp/s/collection1/, http://127.0.0.1:29583/ocp/s/collection1/, http://127.0.0.1:10348/ocp/s/collection1/, http://127.0.0.1:29603/ocp/s/collection1/, http://127.0.0.1:57708/ocp/s/collection1/, http://127.0.0.1:29609/ocp/s/collection1/, http://127.0.0.1:64149/ocp/s/collection1/]</str>   <str name="trace">org.apache.solr.common.SolrException: 7 out of 8 replicas failed to update their schema to version 25 within 15 seconds! Failed cores: [http://127.0.0.1:29575/ocp/s/collection1/, http://127.0.0.1:29583/ocp/s/collection1/, http://127.0.0.1:10348/ocp/s/collection1/, http://127.0.0.1:29603/ocp/s/collection1/, http://127.0.0.1:57708/ocp/s/collection1/, http://127.0.0.1:29609/ocp/s/collection1/, http://127.0.0.1:64149/ocp/s/collection1/]  at org.apache.solr.schema.ManagedIndexSchema.waitForSchemaZkVersionAgreement(ManagedIndexSchema.java:265)  at org.apache.solr.rest.schema.BaseFieldResource.waitForSchemaUpdateToPropagate(BaseFieldResource.java:120)  at org.apache.solr.rest.schema.CopyFieldCollectionResource.post(CopyFieldCollectionResource.java:189)  at org.restlet.resource.ServerResource.doHandle(ServerResource.java:437)  at org.restlet.resource.ServerResource.doConditionalHandle(ServerResource.java:350)  at org.restlet.resource.ServerResource.handle(ServerResource.java:952)  at org.restlet.resource.Finder.handle(Finder.java:246)  at org.restlet.routing.Filter.doHandle(Filter.java:159)  at org.restlet.routing.Filter.handle(Filter.java:206)  at org.restlet.routing.Router.doHandle(Router.java:431)  at org.restlet.routing.Router.handle(Router.java:648)  at org.restlet.routing.Filter.doHandle(Filter.java:159)  at org.restlet.routing.Filter.handle(Filter.java:206)  at org.restlet.routing.Filter.doHandle(Filter.java:159)  at org.restlet.routing.Filter.handle(Filter.java:206)  at org.restlet.routing.Filter.doHandle(Filter.java:159)  at org.restlet.engine.application.StatusFilter.doHandle(StatusFilter.java:155)  at org.restlet.routing.Filter.handle(Filter.java:206)  at org.restlet.routing.Filter.doHandle(Filter.java:159)  at org.restlet.routing.Filter.handle(Filter.java:206)  at org.restlet.engine.CompositeHelper.handle(CompositeHelper.java:211)  at org.restlet.engine.application.ApplicationHelper.handle(ApplicationHelper.java:84)  at org.restlet.Application.handle(Application.java:381)  at org.restlet.routing.Filter.doHandle(Filter.java:159)  at org.restlet.routing.Filter.handle(Filter.java:206)  at org.restlet.routing.Router.doHandle(Router.java:431)  at org.restlet.routing.Router.handle(Router.java:648)  at org.restlet.routing.Filter.doHandle(Filter.java:159)  at org.restlet.routing.Filter.handle(Filter.java:206)  at org.restlet.routing.Router.doHandle(Router.java:431)  at org.restlet.routing.Router.handle(Router.java:648)  at org.restlet.routing.Filter.doHandle(Filter.java:159)  at org.restlet.routing.Filter.handle(Filter.java:206)  at org.restlet.engine.CompositeHelper.handle(CompositeHelper.java:211)  at org.restlet.Component.handle(Component.java:392)  at org.restlet.Server.handle(Server.java:516)  at org.restlet.engine.ServerHelper.handle(ServerHelper.java:72)  at org.restlet.engine.adapter.HttpServerHelper.handle(HttpServerHelper.java:152)  at org.restlet.ext.servlet.ServerServlet.service(ServerServlet.java:1089)  at javax.servlet.http.HttpServlet.service(HttpServlet.java:848)  at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:669)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:457)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)  at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)  at org.eclipse.jetty.server.Dispatcher.forward(Dispatcher.java:276)  at org.eclipse.jetty.server.Dispatcher.forward(Dispatcher.java:103)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:102)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)  at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)  at org.eclipse.jetty.server.Server.handle(Server.java:368)  at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)  at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)  at org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:953)  at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1014)  at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:861)  at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240)  at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)  at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)  at java.lang.Thread.run(Thread.java:745) </str>   <int name="code">500</int> </lst> </response> 

Stack Trace:
java.lang.AssertionError: QUERY FAILED: xpath=/response/lst[@name='responseHeader']/int[@name='status'][.='0']  request=/schema/copyfields/?wt=xml&updateTimeoutSecs=15  response=<?xml version="1.0" encoding="UTF-8"?>
<response>

<lst name="responseHeader">
  <int name="status">500</int>
  <int name="QTime">46117</int>
</lst>
<lst name="error">
  <str name="msg">7 out of 8 replicas failed to update their schema to version 25 within 15 seconds! Failed cores: [http://127.0.0.1:29575/ocp/s/collection1/, http://127.0.0.1:29583/ocp/s/collection1/, http://127.0.0.1:10348/ocp/s/collection1/, http://127.0.0.1:29603/ocp/s/collection1/, http://127.0.0.1:57708/ocp/s/collection1/, http://127.0.0.1:29609/ocp/s/collection1/, http://127.0.0.1:64149/ocp/s/collection1/]</str>
  <str name="trace">org.apache.solr.common.SolrException: 7 out of 8 replicas failed to update their schema to version 25 within 15 seconds! Failed cores: [http://127.0.0.1:29575/ocp/s/collection1/, http://127.0.0.1:29583/ocp/s/collection1/, http://127.0.0.1:10348/ocp/s/collection1/, http://127.0.0.1:29603/ocp/s/collection1/, http://127.0.0.1:57708/ocp/s/collection1/, http://127.0.0.1:29609/ocp/s/collection1/, http://127.0.0.1:64149/ocp/s/collection1/]
	at org.apache.solr.schema.ManagedIndexSchema.waitForSchemaZkVersionAgreement(ManagedIndexSchema.java:265)
	at org.apache.solr.rest.schema.BaseFieldResource.waitForSchemaUpdateToPropagate(BaseFieldResource.java:120)
	at org.apache.solr.rest.schema.CopyFieldCollectionResource.post(CopyFieldCollectionResource.java:189)
	at org.restlet.resource.ServerResource.doHandle(ServerResource.java:437)
	at org.restlet.resource.ServerResource.doConditionalHandle(ServerResource.java:350)
	at org.restlet.resource.ServerResource.handle(ServerResource.java:952)
	at org.restlet.resource.Finder.handle(Finder.java:246)
	at org.restlet.routing.Filter.doHandle(Filter.java:159)
	at org.restlet.routing.Filter.handle(Filter.java:206)
	at org.restlet.routing.Router.doHandle(Router.java:431)
	at org.restlet.routing.Router.handle(Router.java:648)
	at org.restlet.routing.Filter.doHandle(Filter.java:159)
	at org.restlet.routing.Filter.handle(Filter.java:206)
	at org.restlet.routing.Filter.doHandle(Filter.java:159)
	at org.restlet.routing.Filter.handle(Filter.java:206)
	at org.restlet.routing.Filter.doHandle(Filter.java:159)
	at org.restlet.engine.application.StatusFilter.doHandle(StatusFilter.java:155)
	at org.restlet.routing.Filter.handle(Filter.java:206)
	at org.restlet.routing.Filter.doHandle(Filter.java:159)
	at org.restlet.routing.Filter.handle(Filter.java:206)
	at org.restlet.engine.CompositeHelper.handle(CompositeHelper.java:211)
	at org.restlet.engine.application.ApplicationHelper.handle(ApplicationHelper.java:84)
	at org.restlet.Application.handle(Application.java:381)
	at org.restlet.routing.Filter.doHandle(Filter.java:159)
	at org.restlet.routing.Filter.handle(Filter.java:206)
	at org.restlet.routing.Router.doHandle(Router.java:431)
	at org.restlet.routing.Router.handle(Router.java:648)
	at org.restlet.routing.Filter.doHandle(Filter.java:159)
	at org.restlet.routing.Filter.handle(Filter.java:206)
	at org.restlet.routing.Router.doHandle(Router.java:431)
	at org.restlet.routing.Router.handle(Router.java:648)
	at org.restlet.routing.Filter.doHandle(Filter.java:159)
	at org.restlet.routing.Filter.handle(Filter.java:206)
	at org.restlet.engine.CompositeHelper.handle(CompositeHelper.java:211)
	at org.restlet.Component.handle(Component.java:392)
	at org.restlet.Server.handle(Server.java:516)
	at org.restlet.engine.ServerHelper.handle(ServerHelper.java:72)
	at org.restlet.engine.adapter.HttpServerHelper.handle(HttpServerHelper.java:152)
	at org.restlet.ext.servlet.ServerServlet.service(ServerServlet.java:1089)
	at javax.servlet.http.HttpServlet.service(HttpServlet.java:848)
	at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:669)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:457)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
	at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
	at org.eclipse.jetty.server.Dispatcher.forward(Dispatcher.java:276)
	at org.eclipse.jetty.server.Dispatcher.forward(Dispatcher.java:103)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:102)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
	at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
	at org.eclipse.jetty.server.Server.handle(Server.java:368)
	at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
	at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
	at org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:953)
	at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1014)
	at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:861)
	at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240)
	at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
	at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
	at java.lang.Thread.run(Thread.java:745)
</str>
  <int name="code">500</int>
</lst>
</response>

	at __randomizedtesting.SeedInfo.seed([932E7C58AD80ADA2:1B7A4382037CC05A]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.schema.TestCloudManagedSchemaConcurrent.verifySuccess(TestCloudManagedSchemaConcurrent.java:107)
	at org.apache.solr.schema.TestCloudManagedSchemaConcurrent.copyField(TestCloudManagedSchemaConcurrent.java:153)
	at org.apache.solr.schema.TestCloudManagedSchemaConcurrent.access$500(TestCloudManagedSchemaConcurrent.java:45)
	at org.apache.solr.schema.TestCloudManagedSchemaConcurrent$Operation$5.execute(TestCloudManagedSchemaConcurrent.java:320)
	at org.apache.solr.schema.TestCloudManagedSchemaConcurrent.concurrentOperationsTest(TestCloudManagedSchemaConcurrent.java:450)
	at org.apache.solr.schema.TestCloudManagedSchemaConcurrent.test(TestCloudManagedSchemaConcurrent.java:252)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.update.SoftAutoCommitTest

Error Message:
Some resources were not closed, shutdown, or released.

Stack Trace:
java.lang.AssertionError: Some resources were not closed, shutdown, or released.
	at __randomizedtesting.SeedInfo.seed([932E7C58AD80ADA2]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.SolrTestCaseJ4.afterClass(SolrTestCaseJ4.java:234)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:799)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.solr.update.processor.TestNamedUpdateProcessors.test

Error Message:
Error from server at http://127.0.0.1:29369: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 404 Can not find: /admin/collections</title> </head> <body> <h2>HTTP ERROR: 404</h2> <p>Problem accessing /admin/collections. Reason: <pre>    Can not find: /admin/collections</pre></p> <hr /><i><small>Powered by Jetty://</small></i>                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:29369: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 Can not find: /admin/collections</title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /admin/collections. Reason:
<pre>    Can not find: /admin/collections</pre></p>
<hr /><i><small>Powered by Jetty://</small></i>
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([932E7C58AD80ADA2:1B7A4382037CC05A]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:525)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:152)
	at org.apache.solr.handler.TestBlobHandler.createSystemCollection(TestBlobHandler.java:112)
	at org.apache.solr.update.processor.TestNamedUpdateProcessors.test(TestNamedUpdateProcessors.java:86)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.update.processor.TestNamedUpdateProcessors

Error Message:
Some resources were not closed, shutdown, or released.

Stack Trace:
java.lang.AssertionError: Some resources were not closed, shutdown, or released.
	at __randomizedtesting.SeedInfo.seed([932E7C58AD80ADA2]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.SolrTestCaseJ4.afterClass(SolrTestCaseJ4.java:234)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:799)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:33528/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33528/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([932E7C58AD80ADA2:1B7A4382037CC05A]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:33528 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:41489 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:41489 failed to respond
	at __randomizedtesting.SeedInfo.seed([932E7C58AD80ADA2:1B7A4382037CC05A]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:41489 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:24851 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:24851 failed to respond
	at __randomizedtesting.SeedInfo.seed([932E7C58AD80ADA2:1B7A4382037CC05A]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:24851 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9365 lines...]
   [junit4] Suite: org.apache.solr.update.SoftAutoCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest 932E7C58AD80ADA2-001/init-core-data-001
   [junit4]   2> 116045 T295 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 116047 T295 oas.SolrTestCaseJ4.initCore ####initCore
   [junit4]   2> 116048 T295 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 116049 T295 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 116051 T295 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 116053 T295 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 116312 T295 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 116351 T295 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 116422 T295 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 116424 T295 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 116474 T295 oass.IndexSchema.readSchema [null] Schema name=test
   [junit4]   2> 117381 T295 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 117441 T295 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 117447 T295 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 117481 T295 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 117492 T295 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 117503 T295 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 117506 T295 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 117507 T295 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 117507 T295 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 117509 T295 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 117510 T295 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 117510 T295 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 117511 T295 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr
   [junit4]   2> 117511 T295 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/'
   [junit4]   2> 117536 T295 oasc.CoreContainer.<init> New CoreContainer 393660134
   [junit4]   2> 117536 T295 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/]
   [junit4]   2> 117537 T295 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/lib
   [junit4]   2> 117538 T295 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/lib).
   [junit4]   2> 117555 T295 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 600000
   [junit4]   2> 117556 T295 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 117556 T295 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 60000
   [junit4]   2> 117556 T295 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 117557 T295 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 117557 T295 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 117557 T295 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 117557 T295 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 117558 T295 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 117558 T295 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 117558 T295 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 117567 T295 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=30000&connTimeout=30000&retry=true
   [junit4]   2> 117568 T295 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 117568 T295 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 117568 T295 oasc.CoreContainer.load Node Name: testNode
   [junit4]   2> 117571 T295 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=collection1, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest 932E7C58AD80ADA2-001/init-core-data-001, transient=false, shard=shard1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, name=collection1, loadOnStartup=true, instanceDir=collection1, config=solrconfig.xml, schema=schema.xml}
   [junit4]   2> 117572 T296 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 117574 T296 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 117575 T296 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 117575 T296 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 117755 T296 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 117794 T296 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 117906 T296 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 117910 T296 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 117935 T296 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 118927 T296 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 118946 T296 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 118948 T296 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 118966 T296 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 118971 T296 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 118974 T296 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 118975 T296 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 118975 T296 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 118976 T296 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 118976 T296 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 118977 T296 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 118977 T296 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 118977 T296 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 118978 T296 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest 932E7C58AD80ADA2-001/init-core-data-001/
   [junit4]   2> 118978 T296 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@75749e15
   [junit4]   2> 118979 T296 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest 932E7C58AD80ADA2-001/init-core-data-001
   [junit4]   2> 118980 T296 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest 932E7C58AD80ADA2-001/init-core-data-001/index/
   [junit4]   2> 118980 T296 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest 932E7C58AD80ADA2-001/init-core-data-001/index' doesn't exist. Creating new index...
   [junit4]   2> 118981 T296 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest 932E7C58AD80ADA2-001/init-core-data-001/index
   [junit4]   2> 118981 T296 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=38, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.635920085406242]
   [junit4]   2> 118982 T296 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3808195b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1bb8c21a),segFN=segments_1,generation=1}
   [junit4]   2> 118983 T296 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 118992 T296 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 118993 T296 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 118993 T296 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 118994 T296 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 118994 T296 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 118994 T296 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 118995 T296 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 118995 T296 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 118995 T296 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 118996 T296 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 118997 T296 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 118997 T296 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 119025 T296 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 119028 T296 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 119031 T296 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 119034 T296 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 119044 T296 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 119050 T296 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: mltrh,spellCheckCompRH_Direct,standard,/admin/mbeans,/admin/system,/replication,lazy,/update/csv,/update/json,/admin/logging,/debug/dump,/search-facet-invariants,/admin/plugins,/get,/admin/ping,spellCheckWithWordbreak,/admin/segments,spellCheckCompRH1,/admin/file,/mlt,/admin/properties,spellCheckCompRH,/search-facet-def,/admin/threads,tvrh,dismax,/update,spellCheckWithWordbreak_Direct,/schema,/config,defaults,/update/json/docs,mock,/terms,/admin/luke
   [junit4]   2> 119052 T296 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 119055 T296 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 119059 T296 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 119060 T296 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 119061 T296 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 119061 T296 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 119062 T296 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@2d7292af
   [junit4]   2> 119063 T296 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3808195b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1bb8c21a),segFN=segments_1,generation=1}
   [junit4]   2> 119063 T296 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 119064 T296 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@443f7782[collection1] main
   [junit4]   2> 119065 T296 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 119067 T296 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 119067 T296 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 119067 T296 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 119131 T296 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 119251 T296 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 119253 T297 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 119255 T297 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 119255 T297 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 119255 T296 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 119255 T297 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 119258 T297 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 119260 T297 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 119262 T297 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 119262 T295 oas.SolrTestCaseJ4.initCore ####initCore end
   [junit4]   2> 119264 T297 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 119265 T297 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 119268 T297 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@443f7782[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 119270 T295 oas.SolrTestCaseJ4.setUp ###Starting testSoftAndHardCommitMaxTimeRapidAdds
   [junit4]   2> 119271 T295 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 119276 T295 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 119278 T295 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 119280 T295 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 119687 T295 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 119763 T295 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 119891 T295 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 119893 T295 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 119911 T295 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 165494 T295 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 165535 T295 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 165539 T295 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 165563 T295 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 165568 T295 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 165573 T295 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 165575 T295 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 165575 T295 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 165576 T295 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 165577 T295 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 165577 T295 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 165577 T295 oasc.CoreContainer.reload Reloading SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 165578 T295 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest 932E7C58AD80ADA2-001/init-core-data-001/
   [junit4]   2> 165579 T295 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@75749e15
   [junit4]   2> 165580 T295 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest 932E7C58AD80ADA2-001/init-core-data-001/index/
   [junit4]   2> 165592 T295 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 165593 T295 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 165593 T295 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 165594 T295 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 165594 T295 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 165595 T295 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 165595 T295 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 165596 T295 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 165596 T295 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 165597 T295 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 165598 T295 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 165598 T295 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 165621 T295 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 165624 T295 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 165626 T295 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 165629 T295 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 165644 T295 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 165652 T295 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: mltrh,spellCheckCompRH_Direct,standard,/admin/mbeans,/admin/system,/replication,lazy,/update/csv,/update/json,/admin/logging,/debug/dump,/search-facet-invariants,/admin/plugins,/get,/admin/ping,spellCheckWithWordbreak,/admin/segments,spellCheckCompRH1,/admin/file,/mlt,/admin/properties,spellCheckCompRH,/search-facet-def,/admin/threads,tvrh,dismax,/update,spellCheckWithWordbreak_Direct,/schema,/config,defaults,/update/json/docs,mock,/terms,/admin/luke
   [junit4]   2> 165654 T295 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 165656 T295 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 165663 T295 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 165664 T295 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 165665 T295 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@a2b14bd[collection1] main
   [junit4]   2> 165666 T295 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 165666 T295 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 165667 T295 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 165667 T295 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 165670 T295 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 165682 T295 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 165683 T299 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 165684 T299 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 165684 T299 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 165685 T299 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 165685 T299 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 165686 T295 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
   [junit4]   2> 165686 T295 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
   [junit4]   2> 165686 T299 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 165687 T295 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Closing old IndexWriter... core=collection1
   [junit4]   2> 165687 T299 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 165688 T295 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=10, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=18.26953125, floorSegmentMB=1.3505859375, forceMergeDeletesPctAllowed=24.319748176353148, segmentsPerTier=10.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7853453377607684
   [junit4]   2> 165688 T299 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 165689 T295 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3808195b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1bb8c21a),segFN=segments_1,generation=1}
   [junit4]   2> 165690 T299 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 165690 T295 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 165691 T295 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
   [junit4]   2> 165692 T299 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@a2b14bd[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 165693 T295 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@13b7dd80[collection1] main
   [junit4]   2> 165693 T295 c:collection1 oasc.CoreContainer.registerCore replacing core: collection1
   [junit4]   2> 165694 T295 c:collection1 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@6f2248c7
   [junit4]   2> 165694 T295 c:collection1 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=0,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=0,transaction_logs_total_number=0}
   [junit4]   2> 165695 T299 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@13b7dd80[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 165695 T295 c:collection1 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> ASYNC  NEW_CORE C26 name=collection1 org.apache.solr.core.SolrCore@69cea63e
   [junit4]   2> 165782 T295 c:collection1 C26 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5000 (1497655808439091200)]} 0 68
   [junit4]   2> ASYNC  NEW_CORE C27 name=collection1 org.apache.solr.core.SolrCore@69cea63e
   [junit4]   2> 165795 T295 c:collection1 C27 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5001 (1497655808522977280)]} 0 1
   [junit4]   2> 165799 T295 c:collection1 C27 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5002 (1497655808527171584)]} 0 1
   [junit4]   2> 165802 T295 c:collection1 C27 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5003 (1497655808530317312)]} 0 1
   [junit4]   2> 165806 T295 c:collection1 C27 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5004 (1497655808534511616)]} 0 1
   [junit4]   2> 165808 T295 c:collection1 oas.SolrTestCaseJ4.tearDown ###Ending testSoftAndHardCommitMaxTimeRapidAdds
   [junit4]   2> 165815 T295 c:collection1 oas.SolrTestCaseJ4.setUp ###Starting testSoftAndHardCommitMaxTimeMixedAdds
   [junit4]   2> 165815 T295 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 165817 T295 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 165818 T295 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 165819 T295 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 165952 T295 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 165991 T295 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 166068 T295 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 166069 T295 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 166085 T295 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 166283 T301 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 166587 T301 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@35e0327e[collection1] main
   [junit4]   2> 166589 T301 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 166592 T299 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@35e0327e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):C5)))}
   [junit4]   2> 166733 T295 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 166812 T295 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 166817 T295 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 166843 T295 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 166863 T295 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 166874 T295 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 166876 T295 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 166877 T295 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 166877 T295 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 166879 T295 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 166880 T295 c:col

[...truncated too long message...]

TestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 		2	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1643454 T3785 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 6 connections to: http://127.0.0.1:41489/
   [junit4]   2> 1643454 T3785 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:41493/
   [junit4]   2> 1643454 T3785 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:41483/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=932E7C58AD80ADA2 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=in_ID -Dtests.timezone=America/Louisville -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   69.6s J0 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:41489 failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([932E7C58AD80ADA2:1B7A4382037CC05A]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:41489 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1643476 T3785 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 932E7C58AD80ADA2-001
   [junit4]   2> 69616 T3784 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {timestamp=PostingsFormat(name=Memory doPackFST= true), multiDefault=PostingsFormat(name=Memory doPackFST= false), a_t=PostingsFormat(name=Memory doPackFST= true), id=Lucene50(blocksize=128), intDefault=PostingsFormat(name=Memory doPackFST= true), range_facet_l=PostingsFormat(name=Memory doPackFST= true), _version_=PostingsFormat(name=Memory doPackFST= true), text=PostingsFormat(name=LuceneVarGapFixedInterval)}, docValues:{timestamp=DocValuesFormat(name=Direct)}, sim=DefaultSimilarity, locale=in_ID, timezone=America/Louisville
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=185225304,total=304087040
   [junit4]   2> NOTE: All tests run in this JVM: [DateMathParserTest, OverriddenZkACLAndCredentialsProvidersTest, TestGroupingSearch, TestRandomMergePolicy, TestSort, TestDocBasedVersionConstraints, UniqFieldsUpdateProcessorFactoryTest, TestBadConfig, TestObjectReleaseTracker, AutoCommitTest, TestSearchPerf, PluginInfoTest, TestManagedResourceStorage, TestFastOutputStream, FieldMutatingUpdateProcessorTest, NumericFieldsTest, TestStandardQParsers, DistributedFacetPivotLongTailTest, IndexBasedSpellCheckerTest, TestSolrJ, DistributedQueryElevationComponentTest, CursorPagingTest, TestCursorMarkWithoutUniqueKey, TestFieldSortValues, ZkSolrClientTest, SpatialRPTFieldTypeTest, DOMUtilTest, TestStressUserVersions, SliceStateTest, LeaderElectionTest, DeleteLastCustomShardedReplicaTest, SuggestComponentTest, TestQueryTypes, TestFreeTextSuggestions, DirectUpdateHandlerTest, SpellCheckComponentTest, ExternalCollectionsTest, ChangedSchemaMergeTest, TestCharFilters, DeleteReplicaTest, SimpleMLTQParserTest, PolyFieldTest, TestPostingsSolrHighlighter, TestSolrXml, FacetPivotSmallTest, ResponseHeaderTest, TestFieldTypeCollectionResource, AnalysisErrorHandlingTest, TestSolrCoreProperties, TestImplicitCoreProperties, TestExactStatsCache, SimpleCollectionCreateDeleteTest, MultiThreadedOCPTest, TestFileDictionaryLookup, BasicFunctionalityTest, TestMaxScoreQueryParser, TestPivotHelperCode, ResponseLogComponentTest, HttpPartitionTest, TestReplicationHandler, UnloadDistributedZkTest, DistribDocExpirationUpdateProcessorTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J0 in 69.89s, 1 test, 1 error <<< FAILURES!

[...truncated 692 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 2 suite-level errors, 4 errors, 1 failure, 61 ignored (21 assumptions)

Total time: 55 minutes 17 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 217510061 bytes
Compression is 0.0%
Took 42 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2894 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2894/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:48409/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48409/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([A6DBB2D7042F4A95:2E8F8D0DAAD3276D]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:48409 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48539/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48539/collection1
	at __randomizedtesting.SeedInfo.seed([A6DBB2D7042F4A95:2E8F8D0DAAD3276D]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48539/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:48539 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50207/zt_/rp/repfacttest_c8n_1x3_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50207/zt_/rp/repfacttest_c8n_1x3_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([A6DBB2D7042F4A95:2E8F8D0DAAD3276D]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50207/zt_/rp/repfacttest_c8n_1x3_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:50207 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9964 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/init-core-data-001
   [junit4]   2> 1132617 T2416 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /zt_/rp
   [junit4]   2> 1132621 T2416 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1132622 T2417 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1132722 T2416 oasc.ZkTestServer.run start zk server on port:50202
   [junit4]   2> 1132723 T2416 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1132724 T2416 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1132727 T2424 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4ffe3d82 name:ZooKeeperConnection Watcher:127.0.0.1:50202 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1132727 T2416 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1132728 T2416 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1132728 T2416 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1132730 T2416 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1132732 T2416 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1132733 T2427 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@530007a8 name:ZooKeeperConnection Watcher:127.0.0.1:50202/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1132734 T2416 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1132734 T2416 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1132734 T2416 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1132737 T2416 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1132738 T2416 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1132740 T2416 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1132742 T2416 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1132743 T2416 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1132745 T2416 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1132746 T2416 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1132848 T2416 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1132849 T2416 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1132850 T2416 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1132851 T2416 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1132852 T2416 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1132853 T2416 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1132854 T2416 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1132855 T2416 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1132856 T2416 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1132857 T2416 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1132858 T2416 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1132859 T2416 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1132860 T2416 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1132861 T2416 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1132862 T2416 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1132863 T2416 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1132864 T2416 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1132865 T2416 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1134404 T2416 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/cores/collection1
   [junit4]   2> 1134406 T2416 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1134410 T2416 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:50208
   [junit4]   2> 1134410 T2416 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/zt_/rp, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/tempDir-001/control/data, hostPort=50207}
   [junit4]   2> 1134411 T2416 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 1134411 T2416 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/'
   [junit4]   2> 1134444 T2416 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/solr.xml
   [junit4]   2> 1134486 T2416 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/cores
   [junit4]   2> 1134486 T2416 oasc.CoreContainer.<init> New CoreContainer 1897583399
   [junit4]   2> 1134486 T2416 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/]
   [junit4]   2> 1134486 T2416 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/lib
   [junit4]   2> 1134487 T2416 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/lib).
   [junit4]   2> 1134497 T2416 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1134497 T2416 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1134497 T2416 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1134497 T2416 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1134497 T2416 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1134498 T2416 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1134498 T2416 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1134498 T2416 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1134498 T2416 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1134499 T2416 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1134499 T2416 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1134504 T2416 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1134505 T2416 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1134505 T2416 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1134505 T2416 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1134506 T2416 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50202/solr
   [junit4]   2> 1134506 T2416 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1134506 T2416 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1134507 T2416 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1134508 T2439 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@39533a61 name:ZooKeeperConnection Watcher:127.0.0.1:50202 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1134509 T2416 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1134510 T2416 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1134511 T2416 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1134512 T2442 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5211b59d name:ZooKeeperConnection Watcher:127.0.0.1:50202/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1134513 T2416 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1134514 T2416 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1134516 T2416 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1134517 T2416 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1134519 T2416 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1134520 T2416 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1134522 T2416 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1134523 T2416 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1134524 T2416 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1134525 T2416 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50207_zt_%2Frp
   [junit4]   2> 1134525 T2416 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50207_zt_%2Frp
   [junit4]   2> 1134526 T2416 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1134527 T2416 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1134528 T2416 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1134529 T2416 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:50207_zt_%2Frp
   [junit4]   2> 1134529 T2416 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1134530 T2416 oasc.Overseer.start Overseer (id=93602473946251267-127.0.0.1:50207_zt_%2Frp-n_0000000000) starting
   [junit4]   2> 1134532 T2416 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1134536 T2416 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1134536 T2444 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1134536 T2416 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1134536 T2443 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1134539 T2416 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/cores
   [junit4]   2> 1134542 T2416 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/cores/collection1/, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/cores/collection1, shard=, name=collection1, config=solrconfig.xml, transient=false, loadOnStartup=true, collection=control_collection, schema=schema.xml}
   [junit4]   2> 1134542 T2416 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/cores/collection1/
   [junit4]   2> 1134542 T2416 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1134543 T2446 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1134543 T2446 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1134544 T2446 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1134544 T2442 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1134544 T2443 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50207/zt_/rp",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:50207_zt_%2Frp",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null} current state version: 0
   [junit4]   2> 1134545 T2443 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50207/zt_/rp",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:50207_zt_%2Frp",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 1134545 T2443 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1134545 T2443 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1134546 T2442 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1135545 T2446 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1135545 T2446 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1135546 T2446 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1135546 T2446 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1135547 T2446 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1135547 T2446 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/cores/collection1/'
   [junit4]   2> 1135567 T2446 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1135569 T2446 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1135580 T2446 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1135603 T2446 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1135666 T2446 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1135768 T2446 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1135778 T2446 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1136280 T2446 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1136299 T2446 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1136302 T2446 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1136313 T2446 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1136318 T2446 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1136323 T2446 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1136323 T2446 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1136324 T2446 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1136324 T2446 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1136325 T2446 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1136325 T2446 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1136325 T2446 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1136325 T2446 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1136326 T2446 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/cores/collection1/data/
   [junit4]   2> 1136326 T2446 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@143d13d6
   [junit4]   2> 1136327 T2446 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/cores/collection1/data
   [junit4]   2> 1136327 T2446 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/cores/collection1/data/index/
   [junit4]   2> 1136327 T2446 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1136328 T2446 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/cores/collection1/data/index
   [junit4]   2> 1136328 T2446 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=34, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=57.4541015625, floorSegmentMB=0.5546875, forceMergeDeletesPctAllowed=5.22780806596198, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.19405204540464516
   [junit4]   2> 1136329 T2446 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@34e08c6c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2232193),segFN=segments_1,generation=1}
   [junit4]   2> 1136329 T2446 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1136335 T2446 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1136335 T2446 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1136336 T2446 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1136336 T2446 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1136336 T2446 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1136336 T2446 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1136337 T2446 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1136337 T2446 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1136337 T2446 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1136365 T2446 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1136368 T2446 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1136370 T2446 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1136373 T2446 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1136378 T2446 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/system,/admin/logging,/update/csv,/update,/admin/plugins,/update/json,/admin/file,/get,standard,/admin/mbeans,/replication,/admin/threads,/config,/admin/segments,/admin/luke,/admin/properties,/schema,/admin/ping,/update/json/docs
   [junit4]   2> 1136381 T2446 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1136383 T2446 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1136383 T2446 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1136385 T2446 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1136385 T2446 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1136386 T2446 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=36, maxMergedSegmentMB=22.39453125, floorSegmentMB=1.7548828125, forceMergeDeletesPctAllowed=9.803215846184616, segmentsPerTier=36.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1136387 T2446 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@34e08c6c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2232193),segFN=segments_1,generation=1}
   [junit4]   2> 1136387 T2446 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1136387 T2446 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@2929af9d[collection1] main
   [junit4]   2> 1136388 T2446 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1136388 T2446 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1136389 T2446 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1136389 T2446 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1136389 T2446 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1136390 T2446 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1136390 T2446 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1136390 T2446 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1136391 T2446 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1136391 T2446 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1136392 T2447 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2929af9d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1136392 T2446 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1136393 T2450 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50207/zt_/rp collection:control_collection shard:shard1
   [junit4]   2> 1136393 T2416 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 1136394 T2416 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1136394 T2450 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1136396 T2416 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1136396 T2416 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1136398 T2454 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@688cf5a5 name:ZooKeeperConnection Watcher:127.0.0.1:50202/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1136398 T2416 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1136398 T2450 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1136398 T2416 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1136399 T2416 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1136400 T2442 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1136401 T2450 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1136401 T2450 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C428 name=collection1 org.apache.solr.core.SolrCore@115569db url=http://127.0.0.1:50207/zt_/rp/collection1 node=127.0.0.1:50207_zt_%2Frp C428_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:50207/zt_/rp, core=collection1, node_name=127.0.0.1:50207_zt_%2Frp, state=down}
   [junit4]   2> 1136401 T2450 C:control_collection S:shard1 C428 P50207 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:50207/zt_/rp/collection1/
   [junit4]   2> 1136401 T2443 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 1136402 T2416 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1136401 T2450 C:control_collection S:shard1 C428 P50207 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1136402 T2416 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 1136402 T2450 C:control_collection S:shard1 C428 P50207 oasc.SyncStrategy.syncToMe http://127.0.0.1:50207/zt_/rp/collection1/ has no replicas
   [junit4]   2> 1136403 T2442 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1136403 T2450 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:50207/zt_/rp/collection1/ shard1
   [junit4]   2> 1136403 T2454 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1136402 T2416 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1136403 T2450 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1136404 T2416 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1136406 T2457 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@634940ba name:ZooKeeperConnection Watcher:127.0.0.1:50202/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1136406 T2416 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1136406 T2416 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1136408 T2442 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1136409 T2443 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 2, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 1136410 T2443 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 1136412 T2443 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50207/zt_/rp",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 1136412 T2443 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 1136413 T2454 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1136413 T2442 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1136416 T2442 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1136416 T2442 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1136416 T2454 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1136460 T2450 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:50207/zt_/rp/collection1/ and leader is http://127.0.0.1:50207/zt_/rp/collection1/
   [junit4]   2> 1136461 T2450 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:50207/zt_/rp
   [junit4]   2> 1136461 T2450 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1136461 T2450 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1136461 T2450 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1136463 T2442 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1136464 T2443 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50207/zt_/rp",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:50207_zt_%2Frp",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":"shard1"} current state version: 4
   [junit4]   2> 1136464 T2443 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50207/zt_/rp",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:50207_zt_%2Frp",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 1136568 T2442 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1136568 T2454 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1137646 T2416 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/cores/collection1
   [junit4]   2> 1137648 T2416 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001
   [junit4]   2> 1137649 T2416 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1137651 T2416 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:50220
   [junit4]   2> 1137652 T2416 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/zt_/rp, solrconfig=solrconfig.xml, hostPort=50219, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/cores}
   [junit4]   2> 1137652 T2416 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 1137652 T2416 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/'
   [junit4]   2> 1137681 T2416 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/solr.xml
   [junit4]   2> 1137716 T2416 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/cores
   [junit4]   2> 1137717 T2416 oasc.CoreContainer.<init> New CoreContainer 294604378
   [junit4]   2> 1137717 T2416 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/]
   [junit4]   2> 1137717 T2416 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/lib
   [junit4]   2> 1137718 T2416 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/lib).
   [junit4]   2> 1137727 T2416 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1137727 T2416 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1137727 T2416 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1137728 T2416 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1137728 T2416 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1137728 T2416 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1137728 T2416 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1137728 T2416 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1137729 T2416 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1137729 T2416 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1137729 T2416 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1137734 T2416 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1137734 T2416 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1137734 T2416 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1137735 T2416 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1137735 T2416 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50202/solr
   [junit4]   2> 1137735 T2416 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1137735 T2416 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1137736 T2416 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1137738 T2469 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3d722847 name:ZooKeeperConnection Watcher:127.0.0.1:50202 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1137738 T2416 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1137739 T2416 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1137740 T2416 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1137741 T2472 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@116eb63 name:ZooKeeperConnection Watcher:127.0.0.1:50202/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1137742 T2416 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1137745 T2416 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1138749 T2416 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50219_zt_%2Frp
   [junit4]   2> 1138750 T2416 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50219_zt_%2Frp
   [junit4]   2> 1138753 T2416 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1138756 T2416 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/cores
   [junit4]   2> 1138758 T2416 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/cores/collection1/, dataDir=data/, transient=false, loadOnStartup=true, config=solrconfig.xml, coreNodeName=, collection=collection1, shard=, name=collection1, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/cores/collection1}
   [junit4]   2> 1138759 T2416 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/cores/collection1/
   [junit4]   2> 1138759 T2416 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1138761 T2473 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1138761 T2473 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1138762 T2442 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1138762 T2473 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 1138763 T2473 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 1138763 T2443 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50219/zt_/rp",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:50219_zt_%2Frp",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null} current state version: 5
   [junit4]   2> 1138763 T2473 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 1138763 T2443 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50219/zt_/rp",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:50219_zt_%2Frp",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 1138764 T2443 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 1138764 T2443 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1138764 T2473 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 1138764 T2473 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1138765 T2443 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 1138765 T2472 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1138766 T2472 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 1139765 T2473 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1139766 T2473 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1139766 T2473 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1139766 T2473 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1139767 T2473 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1139767 T2473 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/cores/collection1/'
   [junit4]   2> 1139780 T2473 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1139782 T2473 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1139791 T2473 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1139810 T2473 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1139851 T2473 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1139952 T2473 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1139959 T2473 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1140253 T2473 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1140263 T2473 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1140266 T2473 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1140272 T2473 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1140275 T2473 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1140278 T2473 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1140279 T2473 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1140279 T2473 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1140279 T2473 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1140279 T2473 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1140280 T2473 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1140280 T2473 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1140280 T2473 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1140280 T2473 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 1140280 T2473 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@143d13d6
   [junit4]   2> 1140281 T2473 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/cores/collection1/data
   [junit4]   2> 1140281 T2473 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 1140282 T2473 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1140282 T2473 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 1140282 T2473 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=34, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=57.4541015625, floorSegmentMB=0.5546875, forceMergeDeletesPctAllowed=5.22780806596198, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.19405204540464516
   [junit4]   2> 1140283 T2473 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5eff06be lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3913d4ab),segFN=segments_1,generation=1}
   [junit4]   2> 1140283 T2473 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1140288 T2473 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1140288 T2473 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1140288 T2473 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1140288 T2473 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1140288 T2473 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1140288 T2473 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1140289 T2473 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1140289 T2473 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1140289 T2473 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1140301 T2473 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1140303 T2473 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1140304 T2473 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1140306 T2473 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1140311 T2473 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/system,/admin/logging,/update/csv,/update,/admin/plugins,/update/json,/admin/file,/get,standard,/admin/mbeans,/replication,/admin/threads,/config,/admin/segments,/admin/luke,/admin/properties,/schema,/admin/ping,/update/json/docs
   [junit4]   2> 1140312 T2473 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1140313 T2473 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1140313 T2473 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1140314 T2473 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1140314 T2473 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1140315 T2473 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=36, maxMergedSegmentMB=22.39453125, floorSegmentMB=1.7548828125, forceMergeDeletesPctAllowed=9.803215846184616, segmentsPerTier=36.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1140316 T2473 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5eff06be lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3913d4ab),segFN=segments_1,generation=1}
   [junit4]   2> 1140316 T2473 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1140316 T2473 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@1857ee1b[collection1] main
   [junit4]   2> 1140316 T2473 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1140317 T2473 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1140317 T2473 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1140317 T2473 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1140317 T2473 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1140318 T2473 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1140318 T2473 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1140318 T2473 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1140318 T2473 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1140319 T2473 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1140319 T2474 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1857ee1b[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1140320 T2473 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1140321 T2477 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50219/zt_/rp collection:collection1 shard:shard1
   [junit4]   2> 1140321 T2416 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 1140321 T2416 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1140321 T2477 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1140324 T2477 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1140325 T2442 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1140326 T2477 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1140326 T2477 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C429 name=collection1 org.apache.solr.core.SolrCore@796050a8 url=http://127.0.0.1:50219/zt_/rp/collection1 node=127.0.0.1:50219_zt_%2Frp C429_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:50219/zt_/rp, core=collection1, node_name=127.0.0.1:50219_zt_%2Frp, state=down}
   [junit4]   2> 1140326 T2477 C:collection1 S:shard1 C429 P50219 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:50219/zt_/rp/collection1/
   [junit4]   2> 1140326 T2443 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 1140326 T2477 C:collection1 S:shard1 C429 P50219 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1140327 T2477 C:collection1 S:shard1 C429 P50219 oasc.SyncStrategy.syncToMe http://127.0.0.1:50219/zt_/rp/collection1/ has no replicas
   [junit4]   2> 1140327 T2477 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:50219/zt_/rp/collection1/ shard1
   [junit4]   2> 1140327 T2477 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1140327 T2443 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 1140328 T2472 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1140328 T2472 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 1140330 T2443 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50219/zt_/rp",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 1140330 T2443 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 1140331 T2472 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1140331 T2472 oascc.ZkStateRea

[...truncated too long message...]

 		4	/solr/aliases.json
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 		2	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1598486 T4791 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:48552/
   [junit4]   2> 1598486 T4791 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:48526/
   [junit4]   2> 1598486 T4791 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:48539/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=A6DBB2D7042F4A95 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=nl_NL -Dtests.timezone=Asia/Pyongyang -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   66.5s J0 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48539/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([A6DBB2D7042F4A95:2E8F8D0DAAD3276D]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48539/collection1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:48539 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 54 more
   [junit4]   2> 1598502 T4791 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest A6DBB2D7042F4A95-001
   [junit4]   2> 66505 T4790 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {a_t=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), text=PostingsFormat(name=Memory doPackFST= true), intDefault=PostingsFormat(name=LuceneFixedGap), id=Lucene50(blocksize=128), range_facet_l=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), _version_=PostingsFormat(name=LuceneFixedGap), timestamp=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), multiDefault=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128)))}, docValues:{timestamp=DocValuesFormat(name=Asserting)}, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=nl_NL, timezone=Asia/Pyongyang
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=96816168,total=301465600
   [junit4]   2> NOTE: All tests run in this JVM: [PluginInfoTest, SearchHandlerTest, TestNoOpRegenerator, HdfsSyncSliceTest, TestZkChroot, TestElisionMultitermQuery, CoreAdminHandlerTest, HttpPartitionTest, VMParamsZkACLAndCredentialsProvidersTest, SpellCheckCollatorTest, DistanceFunctionTest, TestRTGBase, TestCSVResponseWriter, TestShardHandlerFactory, MigrateRouteKeyTest, TestDefaultStatsCache, TestSolrXml, RequestLoggingTest, TestConfigReload, DirectSolrSpellCheckerTest, TestSortingResponseWriter, DistributedQueueTest, EchoParamsTest, DOMUtilTest, CircularListTest, TestClassNameShortening, ChangedSchemaMergeTest, TestSweetSpotSimilarityFactory, ActionThrottleTest, TestFastOutputStream, CustomCollectionTest, TermVectorComponentDistributedTest, SpellCheckComponentTest, TestDistributedMissingSort, CacheHeaderTest, MultiTermTest, TestBulkSchemaAPI, TermVectorComponentTest, TestSolrIndexConfig, TestSystemIdResolver, CurrencyFieldOpenExchangeTest, BlockDirectoryTest, DeleteReplicaTest, LoggingHandlerTest, ReplicationFactorTest, TestExactSharedStatsCache, TestSimpleQParserPlugin, TestMissingGroups, TestStressLucene, SolrXmlInZkTest, SpatialHeatmapFacetsTest, UpdateRequestProcessorFactoryTest, SolrCoreTest, SharedFSAutoReplicaFailoverTest, SolrTestCaseJ4Test, TestLeaderElectionZkExpiry, TestNamedUpdateProcessors, MultiThreadedOCPTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J0 in 67.04s, 1 test, 1 error <<< FAILURES!

[...truncated 691 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 54 minutes 6 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 212405397 bytes
Compression is 0.0%
Took 42 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2893 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2893/

5 tests failed.
REGRESSION:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
There were too many update fails (39 > 20) - we expect it can happen, but shouldn't easily

Stack Trace:
java.lang.AssertionError: There were too many update fails (39 > 20) - we expect it can happen, but shouldn't easily
	at __randomizedtesting.SeedInfo.seed([25E72E934121CB2B:ADB31149EFDDA6D3]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertFalse(Assert.java:68)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:230)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.solr.handler.component.DistributedMLTComponentTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:37002//collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:37002//collection1
	at __randomizedtesting.SeedInfo.seed([25E72E934121CB2B:ADB31149EFDDA6D3]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:566)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:558)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:606)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:588)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:567)
	at org.apache.solr.handler.component.DistributedMLTComponentTest.test(DistributedMLTComponentTest.java:126)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 51 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:22086/pd/l/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22086/pd/l/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([25E72E934121CB2B:ADB31149EFDDA6D3]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:22086 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:38235/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:38235/collection1
	at __randomizedtesting.SeedInfo.seed([25E72E934121CB2B:ADB31149EFDDA6D3]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:38235/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:38235 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:20826/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:20826/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([25E72E934121CB2B:ADB31149EFDDA6D3]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:20826/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:20826 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9549 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/init-core-data-001
   [junit4]   2> 380058 T1111 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /pd/l
   [junit4]   2> 380066 T1111 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 380066 T1112 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 380166 T1111 oasc.ZkTestServer.run start zk server on port:22062
   [junit4]   2> 380167 T1111 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 380168 T1111 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 380195 T1119 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2f3ded81 name:ZooKeeperConnection Watcher:127.0.0.1:22062 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 380196 T1111 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 380196 T1111 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 380196 T1111 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 380199 T1111 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 380200 T1111 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 380201 T1122 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7337503e name:ZooKeeperConnection Watcher:127.0.0.1:22062/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 380201 T1111 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 380202 T1111 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 380202 T1111 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 380205 T1111 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 380206 T1111 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 380207 T1111 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 380209 T1111 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 380210 T1111 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 380212 T1111 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 380213 T1111 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 380215 T1111 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 380216 T1111 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 380218 T1111 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 380218 T1111 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 380220 T1111 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 380221 T1111 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 380222 T1111 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 380223 T1111 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 380224 T1111 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 380225 T1111 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 380227 T1111 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 380227 T1111 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 380229 T1111 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 380230 T1111 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 380231 T1111 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 380232 T1111 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 380234 T1111 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 380234 T1111 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 381843 T1111 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/cores/collection1
   [junit4]   2> 381848 T1111 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 381852 T1111 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:22069
   [junit4]   2> 381853 T1111 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/cores, hostContext=/pd/l, hostPort=22068, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/tempDir-001/control/data}
   [junit4]   2> 381853 T1111 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 381854 T1111 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/'
   [junit4]   2> 382072 T1111 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/solr.xml
   [junit4]   2> 382126 T1111 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/cores
   [junit4]   2> 382126 T1111 oasc.CoreContainer.<init> New CoreContainer 1645308729
   [junit4]   2> 382127 T1111 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/]
   [junit4]   2> 382127 T1111 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/lib
   [junit4]   2> 382128 T1111 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/lib).
   [junit4]   2> 382137 T1111 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 382137 T1111 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 382137 T1111 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 382137 T1111 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 382138 T1111 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 382138 T1111 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 382138 T1111 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 382138 T1111 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 382138 T1111 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 382139 T1111 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 382139 T1111 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 382154 T1111 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 382154 T1111 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 382155 T1111 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 382155 T1111 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 382155 T1111 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:22062/solr
   [junit4]   2> 382155 T1111 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 382155 T1111 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 382156 T1111 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 382158 T1134 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@62e48bd5 name:ZooKeeperConnection Watcher:127.0.0.1:22062 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 382159 T1111 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 382160 T1111 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 382162 T1111 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 382164 T1137 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3f1d425d name:ZooKeeperConnection Watcher:127.0.0.1:22062/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 382164 T1111 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 382166 T1111 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 382168 T1111 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 382170 T1111 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 382173 T1111 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 382175 T1111 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 382177 T1111 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 382179 T1111 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 382180 T1111 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 382181 T1111 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:22068_pd%2Fl
   [junit4]   2> 382182 T1111 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:22068_pd%2Fl
   [junit4]   2> 382184 T1111 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 382185 T1111 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 382187 T1111 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 382188 T1111 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:22068_pd%2Fl
   [junit4]   2> 382189 T1111 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 382190 T1111 oasc.Overseer.start Overseer (id=93599020707741699-127.0.0.1:22068_pd%2Fl-n_0000000000) starting
   [junit4]   2> 382192 T1111 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 382198 T1111 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 382198 T1139 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 382198 T1111 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 382199 T1138 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 382203 T1111 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/cores
   [junit4]   2> 382206 T1111 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/cores/collection1/, dataDir=data/, loadOnStartup=true, shard=, collection=control_collection, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/cores/collection1, transient=false, schema=schema.xml, coreNodeName=, name=collection1}
   [junit4]   2> 382206 T1111 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/cores/collection1/
   [junit4]   2> 382206 T1111 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 382208 T1141 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 382208 T1141 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 382209 T1137 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 382209 T1141 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 382210 T1138 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22068/pd/l",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:22068_pd%2Fl",
   [junit4]   2> 	  "numShards":"1"} current state version: 0
   [junit4]   2> 382211 T1138 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22068/pd/l",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:22068_pd%2Fl",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 382211 T1138 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 382211 T1138 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 382212 T1137 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 383210 T1141 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 383211 T1141 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 383211 T1141 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 383212 T1141 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 383212 T1141 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 383212 T1141 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/cores/collection1/'
   [junit4]   2> 383241 T1141 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 383243 T1141 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 383251 T1141 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 383266 T1141 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 383311 T1141 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 383312 T1141 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 383326 T1141 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 383729 T1141 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 383746 T1141 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 383757 T1141 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 383773 T1141 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 383782 T1141 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 383786 T1141 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 383787 T1141 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 383787 T1141 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 383787 T1141 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 383788 T1141 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 383788 T1141 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 383788 T1141 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 383789 T1141 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 383789 T1141 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/cores/collection1/data/
   [junit4]   2> 383789 T1141 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6ad957a7
   [junit4]   2> 383790 T1141 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/cores/collection1/data
   [junit4]   2> 383791 T1141 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/cores/collection1/data/index/
   [junit4]   2> 383791 T1141 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 383791 T1141 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/cores/collection1/data/index
   [junit4]   2> 383792 T1141 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=33, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 383793 T1141 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5f7ee311 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@36da212),segFN=segments_1,generation=1}
   [junit4]   2> 383793 T1141 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 383798 T1141 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 383799 T1141 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 383799 T1141 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 383799 T1141 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 383800 T1141 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 383800 T1141 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 383800 T1141 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 383801 T1141 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 383801 T1141 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 383817 T1141 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 383823 T1141 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 383825 T1141 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 383826 T1141 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 383830 T1141 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update,/admin/logging,standard,/replication,/update/json,/admin/system,/admin/threads,/schema,/admin/segments,/admin/plugins,/update/csv,/admin/mbeans,/admin/properties,/get,/config,/update/json/docs,/admin/ping,/admin/luke,/admin/file
   [junit4]   2> 383842 T1141 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 383844 T1141 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 383844 T1141 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 383846 T1141 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 383846 T1141 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 383846 T1141 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=25, maxMergedSegmentMB=75.1943359375, floorSegmentMB=1.021484375, forceMergeDeletesPctAllowed=10.720232696799917, segmentsPerTier=23.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8897883915405346
   [junit4]   2> 383847 T1141 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5f7ee311 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@36da212),segFN=segments_1,generation=1}
   [junit4]   2> 383847 T1141 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 383848 T1141 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@632703e[collection1] main
   [junit4]   2> 383848 T1141 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 383849 T1141 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 383849 T1141 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 383849 T1141 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 383850 T1141 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 383850 T1141 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 383850 T1141 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 383851 T1141 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 383851 T1141 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 383851 T1141 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 383852 T1141 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 383852 T1142 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@632703e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 383853 T1145 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:22068/pd/l collection:control_collection shard:shard1
   [junit4]   2> 383853 T1111 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 383854 T1111 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 383854 T1145 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 383857 T1111 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 383858 T1145 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 383858 T1111 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 383859 T1137 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 383860 T1145 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 383860 T1149 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@298a5b60 name:ZooKeeperConnection Watcher:127.0.0.1:22062/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 383860 T1145 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 383860 T1138 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C274 name=collection1 org.apache.solr.core.SolrCore@293a7fdb url=http://127.0.0.1:22068/pd/l/collection1 node=127.0.0.1:22068_pd%2Fl C274_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:22068/pd/l, core=collection1, node_name=127.0.0.1:22068_pd%2Fl}
   [junit4]   2> 383860 T1145 C:control_collection S:shard1 C274 P22068 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:22068/pd/l/collection1/
   [junit4]   2> 383860 T1111 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 383861 T1145 C:control_collection S:shard1 C274 P22068 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 383861 T1145 C:control_collection S:shard1 C274 P22068 oasc.SyncStrategy.syncToMe http://127.0.0.1:22068/pd/l/collection1/ has no replicas
   [junit4]   2> 383861 T1111 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 383861 T1145 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:22068/pd/l/collection1/ shard1
   [junit4]   2> 383862 T1137 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 383861 T1111 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 383862 T1145 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 383865 T1111 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 383866 T1137 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 383866 T1138 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22068/pd/l",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 383969 T1137 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 383969 T1149 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 384018 T1145 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:22068/pd/l/collection1/ and leader is http://127.0.0.1:22068/pd/l/collection1/
   [junit4]   2> 384019 T1145 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:22068/pd/l
   [junit4]   2> 384019 T1145 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 384019 T1145 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 384019 T1145 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 384021 T1137 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 384022 T1138 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22068/pd/l",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:22068_pd%2Fl",
   [junit4]   2> 	  "numShards":"1"} current state version: 3
   [junit4]   2> 384022 T1138 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22068/pd/l",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:22068_pd%2Fl",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 384125 T1137 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 384125 T1149 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 385499 T1111 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/cores/collection1
   [junit4]   2> 385504 T1111 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001
   [junit4]   2> 385505 T1111 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 385509 T1111 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:22087
   [junit4]   2> 385510 T1111 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=22086, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/cores, solrconfig=solrconfig.xml, hostContext=/pd/l, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/tempDir-001/jetty1}
   [junit4]   2> 385510 T1111 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 385511 T1111 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/'
   [junit4]   2> 385557 T1111 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/solr.xml
   [junit4]   2> 385618 T1111 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/cores
   [junit4]   2> 385619 T1111 oasc.CoreContainer.<init> New CoreContainer 1755072217
   [junit4]   2> 385619 T1111 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/]
   [junit4]   2> 385619 T1111 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/lib
   [junit4]   2> 385620 T1111 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/lib).
   [junit4]   2> 385639 T1111 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 385639 T1111 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 385639 T1111 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 385640 T1111 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 385640 T1111 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 385641 T1111 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 385641 T1111 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 385642 T1111 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 385642 T1111 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 385642 T1111 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 385643 T1111 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 385650 T1111 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 385651 T1111 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 385651 T1111 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 385652 T1111 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 385652 T1111 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:22062/solr
   [junit4]   2> 385652 T1111 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 385653 T1111 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 385654 T1111 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 385655 T1161 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@78134d4c name:ZooKeeperConnection Watcher:127.0.0.1:22062 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 385656 T1111 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 385657 T1111 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 385660 T1111 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 385662 T1164 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2f6887b3 name:ZooKeeperConnection Watcher:127.0.0.1:22062/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 385662 T1111 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 385667 T1111 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 386671 T1111 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:22086_pd%2Fl
   [junit4]   2> 386672 T1111 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:22086_pd%2Fl
   [junit4]   2> 386675 T1111 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 386677 T1111 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/cores
   [junit4]   2> 386681 T1111 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {loadOnStartup=true, coreNodeName=, name=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/cores/collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/cores/collection1/, transient=false, shard=, dataDir=data/, collection=collection1, config=solrconfig.xml, schema=schema.xml}
   [junit4]   2> 386681 T1111 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/cores/collection1/
   [junit4]   2> 386682 T1111 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 386683 T1165 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 386684 T1165 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 386685 T1165 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 386685 T1137 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 386686 T1138 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22086/pd/l",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:22086_pd%2Fl",
   [junit4]   2> 	  "numShards":"1"} current state version: 4
   [junit4]   2> 386686 T1138 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22086/pd/l",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:22086_pd%2Fl",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 386686 T1138 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 386687 T1138 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 386688 T1164 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 386688 T1149 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 386688 T1137 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 387685 T1165 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 387686 T1165 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 387686 T1165 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 387687 T1165 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 387687 T1165 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 387688 T1165 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/cores/collection1/'
   [junit4]   2> 387705 T1165 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 387707 T1165 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 387719 T1165 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 387745 T1165 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 387798 T1165 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 387799 T1165 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 387819 T1165 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 388122 T1165 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 388140 T1165 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 388142 T1165 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 388151 T1165 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 388154 T1165 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 388157 T1165 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 388158 T1165 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 388158 T1165 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 388158 T1165 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 388159 T1165 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 388159 T1165 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 388159 T1165 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 388159 T1165 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 388159 T1165 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 388160 T1165 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6ad957a7
   [junit4]   2> 388160 T1165 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/cores/collection1/data
   [junit4]   2> 388161 T1165 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 388161 T1165 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 388161 T1165 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 388161 T1165 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=33, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 388162 T1165 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@542169c0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5da5e766),segFN=segments_1,generation=1}
   [junit4]   2> 388162 T1165 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 388167 T1165 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init cr

[...truncated too long message...]

934121CB2B-001/tempDir-001/shard2/cores/collection1/data;done=false>>]
   [junit4]   2> 2195371 T6981 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 25E72E934121CB2B-001/tempDir-001/shard2/cores/collection1/data
   [junit4]   2> 2195372 T6981 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 25E72E934121CB2B-001/tempDir-001/shard2/cores/collection1/data/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 25E72E934121CB2B-001/tempDir-001/shard2/cores/collection1/data/index;done=false>>]
   [junit4]   2> 2195372 T6981 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 25E72E934121CB2B-001/tempDir-001/shard2/cores/collection1/data/index
   [junit4]   2> 2195373 T6981 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=DistributedMLTComponentTest -Dtests.method=test -Dtests.seed=25E72E934121CB2B -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=sr_CS -Dtests.timezone=Etc/GMT+1 -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR    102s J3 | DistributedMLTComponentTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:37002//collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([25E72E934121CB2B:ADB31149EFDDA6D3]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:566)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:558)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:606)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:588)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:567)
   [junit4]    > 	at org.apache.solr.handler.component.DistributedMLTComponentTest.test(DistributedMLTComponentTest.java:126)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: java.net.SocketTimeoutException: Read timed out
   [junit4]    > 	at java.net.SocketInputStream.socketRead0(Native Method)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:152)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:122)
   [junit4]    > 	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
   [junit4]    > 	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
   [junit4]    > 	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 51 more
   [junit4]   2> 2195581 T6981 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 25E72E934121CB2B-001
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=sr_CS, timezone=Etc/GMT+1
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=229360800,total=411566080
   [junit4]   2> NOTE: All tests run in this JVM: [EchoParamsTest, RequestHandlersTest, TestIndexingPerformance, ShowFileRequestHandlerTest, BasicFunctionalityTest, StatsComponentTest, IndexSchemaRuntimeFieldTest, TestSort, CoreMergeIndexesAdminHandlerTest, TestCloudInspectUtil, PrimUtilsTest, TestSearchPerf, TestSolrDeletionPolicy2, LeaderElectionIntegrationTest, BadIndexSchemaTest, AtomicUpdatesTest, TestManagedSchemaDynamicFieldResource, TestRealTimeGet, TestReversedWildcardFilterFactory, TestExtendedDismaxParser, BasicDistributedZkTest, TestHdfsUpdateLog, TestSerializedLuceneMatchVersion, BasicDistributedZk2Test, TestCloudManagedSchema, TestSolrJ, TestQuerySenderNoQuery, TestStressVersions, SimpleCollectionCreateDeleteTest, TestLRUStatsCache, HdfsDirectoryFactoryTest, DistributedTermsComponentTest, WordBreakSolrSpellCheckerTest, TestFoldingMultitermQuery, DistributedSpellCheckComponentTest, TestBlendedInfixSuggestions, TestIndexSearcher, DocValuesMissingTest, AsyncMigrateRouteKeyTest, TestManagedSynonymFilterFactory, QueryEqualityTest, CollectionsAPIAsyncDistributedZkTest, TestManagedSchema, JSONWriterTest, TestDocumentBuilder, ChangedSchemaMergeTest, TimeZoneUtilsTest, MergeStrategyTest, HighlighterTest, TestRemoteStreaming, DateMathParserTest, TestEmbeddedSolrServerConstructors, SolrInfoMBeanTest, TestHashPartitioner, CloudExitableDirectoryReaderTest, TestLuceneMatchVersion, CollectionsAPIDistributedZkTest, TestCloudManagedSchemaConcurrent, TestCloudSchemaless, UnloadDistributedZkTest, TestBlobHandler, MigrateRouteKeyTest, DistribJoinFromCollectionTest, ChaosMonkeySafeLeaderTest, TestDistribDocBasedVersion, DistributedFacetPivotLargeTest, TestDistributedMissingSort, DistributedFacetPivotWhiteBoxTest, DistributedMLTComponentTest]
   [junit4] Completed on J3 in 102.46s, 1 test, 1 error <<< FAILURES!

[...truncated 525 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 4 errors, 1 failure, 61 ignored (21 assumptions)

Total time: 58 minutes 17 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 226560664 bytes
Compression is 0.0%
Took 46 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2892 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2892/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:59918/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:59918/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([E99E0FB8B730BCED:61CA306219CCD115]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:59918 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:43674/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:43674/collection1
	at __randomizedtesting.SeedInfo.seed([E99E0FB8B730BCED:61CA306219CCD115]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:43674/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:43674 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:59962/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:59962/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([E99E0FB8B730BCED:61CA306219CCD115]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:59962/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:59962 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 10024 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/init-core-data-001
   [junit4]   2> 1396045 T5918 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 1396049 T5918 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1396050 T5919 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1396150 T5918 oasc.ZkTestServer.run start zk server on port:59906
   [junit4]   2> 1396151 T5918 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1396152 T5918 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1396155 T5926 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3b5e611f name:ZooKeeperConnection Watcher:127.0.0.1:59906 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1396155 T5918 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1396155 T5918 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1396156 T5918 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1396157 T5918 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1396158 T5918 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1396160 T5929 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@40502e3a name:ZooKeeperConnection Watcher:127.0.0.1:59906/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1396160 T5918 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1396160 T5918 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1396160 T5918 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1396162 T5918 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1396163 T5918 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1396164 T5918 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1396165 T5918 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1396166 T5918 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1396168 T5918 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1396168 T5918 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1396271 T5918 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1396271 T5918 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1396273 T5918 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1396273 T5918 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1396274 T5918 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1396275 T5918 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1396276 T5918 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1396277 T5918 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1396278 T5918 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1396278 T5918 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1396280 T5918 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1396280 T5918 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1396281 T5918 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1396282 T5918 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1396283 T5918 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1396284 T5918 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1396285 T5918 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1396285 T5918 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1397467 T5918 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/cores/collection1
   [junit4]   2> 1397470 T5918 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1397474 T5918 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:59911
   [junit4]   2> 1397474 T5918 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/cores, hostContext=/, hostPort=59910}
   [junit4]   2> 1397475 T5918 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 1397475 T5918 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/'
   [junit4]   2> 1397510 T5918 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/solr.xml
   [junit4]   2> 1397558 T5918 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/cores
   [junit4]   2> 1397558 T5918 oasc.CoreContainer.<init> New CoreContainer 806518256
   [junit4]   2> 1397558 T5918 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/]
   [junit4]   2> 1397559 T5918 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/lib
   [junit4]   2> 1397559 T5918 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/lib).
   [junit4]   2> 1397569 T5918 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1397569 T5918 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1397570 T5918 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1397570 T5918 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1397570 T5918 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1397570 T5918 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1397570 T5918 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1397571 T5918 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1397571 T5918 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1397571 T5918 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1397571 T5918 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1397579 T5918 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1397579 T5918 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1397580 T5918 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1397580 T5918 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1397580 T5918 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:59906/solr
   [junit4]   2> 1397580 T5918 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1397581 T5918 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1397581 T5918 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1397583 T5941 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3e2ee70f name:ZooKeeperConnection Watcher:127.0.0.1:59906 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1397583 T5918 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1397583 T5918 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1397585 T5918 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1397586 T5944 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@280478a3 name:ZooKeeperConnection Watcher:127.0.0.1:59906/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1397586 T5918 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1397587 T5918 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1397588 T5918 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1397590 T5918 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1397591 T5918 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1397593 T5918 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1397594 T5918 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1397596 T5918 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1397597 T5918 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1397597 T5918 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59910_
   [junit4]   2> 1397598 T5918 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:59910_
   [junit4]   2> 1397599 T5918 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1397600 T5918 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1397601 T5918 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1397602 T5918 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:59910_
   [junit4]   2> 1397602 T5918 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1397603 T5918 oasc.Overseer.start Overseer (id=93598135630954499-127.0.0.1:59910_-n_0000000000) starting
   [junit4]   2> 1397605 T5918 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1397609 T5918 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1397610 T5946 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1397610 T5918 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1397610 T5945 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1397613 T5918 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/cores
   [junit4]   2> 1397615 T5918 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {coreNodeName=, dataDir=data/, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/cores/collection1, schema=schema.xml, loadOnStartup=true, shard=, name=collection1, transient=false, collection=control_collection, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/cores/collection1/}
   [junit4]   2> 1397615 T5918 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/cores/collection1/
   [junit4]   2> 1397616 T5918 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1397616 T5948 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1397616 T5948 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1397617 T5948 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1397617 T5944 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1397618 T5945 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59910",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:59910_",
   [junit4]   2> 	  "roles":null} current state version: 0
   [junit4]   2> 1397618 T5945 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59910",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:59910_",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 1397619 T5945 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1397619 T5945 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1397620 T5944 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1398617 T5948 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1398618 T5948 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1398618 T5948 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1398618 T5948 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1398619 T5948 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1398619 T5948 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/cores/collection1/'
   [junit4]   2> 1398631 T5948 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1398633 T5948 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1398640 T5948 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1398656 T5948 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1398715 T5948 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1398817 T5948 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1398825 T5948 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1399173 T5948 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1399187 T5948 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1399189 T5948 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1399196 T5948 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1399200 T5948 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1399202 T5948 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1399203 T5948 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1399203 T5948 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1399203 T5948 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1399204 T5948 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1399204 T5948 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1399204 T5948 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1399204 T5948 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1399205 T5948 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/cores/collection1/data/
   [junit4]   2> 1399205 T5948 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3fb7bec8
   [junit4]   2> 1399206 T5948 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/cores/collection1/data
   [junit4]   2> 1399206 T5948 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/cores/collection1/data/index/
   [junit4]   2> 1399206 T5948 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1399206 T5948 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/cores/collection1/data/index
   [junit4]   2> 1399207 T5948 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=29, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8275799343616196]
   [junit4]   2> 1399207 T5948 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5d65f114 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4fb687f3),segFN=segments_1,generation=1}
   [junit4]   2> 1399207 T5948 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1399211 T5948 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1399212 T5948 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1399212 T5948 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1399212 T5948 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1399212 T5948 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1399212 T5948 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1399213 T5948 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1399213 T5948 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1399213 T5948 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1399228 T5948 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1399230 T5948 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1399232 T5948 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1399233 T5948 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1399237 T5948 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/mbeans,/get,/update,/admin/segments,/admin/ping,/admin/properties,/update/json,/update/json/docs,/replication,/admin/threads,/update/csv,/admin/logging,/config,/admin/file,standard,/admin/luke,/admin/plugins,/schema,/admin/system
   [junit4]   2> 1399239 T5948 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1399240 T5948 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1399240 T5948 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1399241 T5948 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1399241 T5948 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1399242 T5948 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=923759317, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1399242 T5948 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5d65f114 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4fb687f3),segFN=segments_1,generation=1}
   [junit4]   2> 1399243 T5948 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1399243 T5948 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@1688003a[collection1] main
   [junit4]   2> 1399243 T5948 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1399244 T5948 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1399244 T5948 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1399244 T5948 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1399245 T5948 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1399245 T5948 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1399245 T5948 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1399246 T5948 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1399246 T5948 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1399246 T5948 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1399247 T5949 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1688003a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1399247 T5948 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1399248 T5952 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:59910 collection:control_collection shard:shard1
   [junit4]   2> 1399248 T5918 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 1399248 T5918 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1399249 T5952 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1399251 T5952 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1399252 T5944 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1399253 T5952 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1399252 T5918 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1399253 T5952 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1399253 T5945 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 1399253 T5918 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> ASYNC  NEW_CORE C840 name=collection1 org.apache.solr.core.SolrCore@23476853 url=http://127.0.0.1:59910/collection1 node=127.0.0.1:59910_ C840_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:59910, core=collection1, node_name=127.0.0.1:59910_}
   [junit4]   2> 1399253 T5952 C:control_collection S:shard1 C840 P59910 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:59910/collection1/
   [junit4]   2> 1399254 T5952 C:control_collection S:shard1 C840 P59910 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1399254 T5952 C:control_collection S:shard1 C840 P59910 oasc.SyncStrategy.syncToMe http://127.0.0.1:59910/collection1/ has no replicas
   [junit4]   2> 1399254 T5952 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:59910/collection1/ shard1
   [junit4]   2> 1399255 T5952 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1399255 T5944 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1399255 T5956 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@668a2c0b name:ZooKeeperConnection Watcher:127.0.0.1:59906/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1399255 T5918 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1399256 T5918 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1399256 T5918 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1399258 T5944 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1399258 T5918 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1399258 T5945 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59910",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 1399361 T5956 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1399361 T5944 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1399410 T5952 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:59910/collection1/ and leader is http://127.0.0.1:59910/collection1/
   [junit4]   2> 1399411 T5952 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:59910
   [junit4]   2> 1399411 T5952 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1399411 T5952 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1399411 T5952 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1399412 T5944 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1399413 T5945 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59910",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:59910_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1"} current state version: 3
   [junit4]   2> 1399414 T5945 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59910",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:59910_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 1399516 T5956 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1399516 T5944 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1400340 T5918 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/cores/collection1
   [junit4]   2> 1400342 T5918 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001
   [junit4]   2> 1400343 T5918 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1400345 T5918 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:59927
   [junit4]   2> 1400345 T5918 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostPort=59926, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/cores}
   [junit4]   2> 1400346 T5918 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 1400346 T5918 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/'
   [junit4]   2> 1400377 T5918 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/solr.xml
   [junit4]   2> 1400415 T5918 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/cores
   [junit4]   2> 1400416 T5918 oasc.CoreContainer.<init> New CoreContainer 477898052
   [junit4]   2> 1400416 T5918 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/]
   [junit4]   2> 1400416 T5918 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/lib
   [junit4]   2> 1400417 T5918 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/lib).
   [junit4]   2> 1400426 T5918 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1400427 T5918 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1400427 T5918 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1400427 T5918 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1400427 T5918 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1400428 T5918 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1400428 T5918 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1400428 T5918 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1400428 T5918 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1400428 T5918 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1400429 T5918 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1400434 T5918 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1400435 T5918 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1400435 T5918 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1400435 T5918 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1400436 T5918 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:59906/solr
   [junit4]   2> 1400436 T5918 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1400436 T5918 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1400437 T5918 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1400438 T5968 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1b8bb05d name:ZooKeeperConnection Watcher:127.0.0.1:59906 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1400439 T5918 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1400439 T5918 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1400440 T5918 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1400441 T5971 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@d77b265 name:ZooKeeperConnection Watcher:127.0.0.1:59906/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1400441 T5918 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1400444 T5918 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1401447 T5918 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59926_
   [junit4]   2> 1401448 T5918 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:59926_
   [junit4]   2> 1401450 T5918 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1401452 T5918 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/cores
   [junit4]   2> 1401455 T5918 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {transient=false, shard=, name=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/cores/collection1, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/cores/collection1/, collection=collection1, coreNodeName=, dataDir=data/, loadOnStartup=true, schema=schema.xml}
   [junit4]   2> 1401455 T5918 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/cores/collection1/
   [junit4]   2> 1401455 T5918 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1401456 T5972 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1401456 T5972 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1401457 T5972 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1401457 T5944 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1401458 T5945 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59926",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:59926_",
   [junit4]   2> 	  "roles":null} current state version: 4
   [junit4]   2> 1401459 T5945 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59926",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:59926_",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 1401459 T5945 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 1401459 T5945 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1401460 T5944 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1401460 T5971 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1401460 T5956 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1402457 T5972 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1402458 T5972 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1402458 T5972 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1402459 T5972 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1402459 T5972 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1402459 T5972 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/cores/collection1/'
   [junit4]   2> 1402472 T5972 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1402474 T5972 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1402482 T5972 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1402500 T5972 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1402540 T5972 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1402642 T5972 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1402653 T5972 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1403092 T5972 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1403109 T5972 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1403113 T5972 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1403121 T5972 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1403126 T5972 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1403130 T5972 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1403131 T5972 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1403131 T5972 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1403131 T5972 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1403132 T5972 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1403132 T5972 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1403132 T5972 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1403133 T5972 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1403133 T5972 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 1403133 T5972 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3fb7bec8
   [junit4]   2> 1403134 T5972 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/cores/collection1/data
   [junit4]   2> 1403135 T5972 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 1403135 T5972 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1403135 T5972 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 1403136 T5972 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=29, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8275799343616196]
   [junit4]   2> 1403136 T5972 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@20c962d5 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5779ec4),segFN=segments_1,generation=1}
   [junit4]   2> 1403137 T5972 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1403143 T5972 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1403143 T5972 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1403144 T5972 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1403144 T5972 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1403144 T5972 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1403144 T5972 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1403144 T5972 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1403145 T5972 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1403145 T5972 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1403162 T5972 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1403163 T5972 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1403165 T5972 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1403167 T5972 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1403173 T5972 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/mbeans,/get,/update,/admin/segments,/admin/ping,/admin/properties,/update/json,/update/json/docs,/replication,/admin/threads,/update/csv,/admin/logging,/config,/admin/file,standard,/admin/luke,/admin/plugins,/schema,/admin/system
   [junit4]   2> 1403174 T5972 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1403176 T5972 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1403176 T5972 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1403177 T5972 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1403177 T5972 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1403179 T5972 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=923759317, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1403179 T5972 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@20c962d5 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5779ec4),segFN=segments_1,generation=1}
   [junit4]   2> 1403180 T5972 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1403180 T5972 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@b85b05d[collection1] main
   [junit4]   2> 1403180 T5972 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1403181 T5972 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1403181 T5972 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1403181 T5972 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1403181 T5972 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1403182 T5972 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1403182 T5972 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1403182 T5972 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1403182 T5972 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1403182 T5972 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1403183 T5973 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@b85b05d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1403184 T5972 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1403184 T5976 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:59926 collection:collection1 shard:shard1
   [junit4]   2> 1403185 T5918 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 1403185 T5918 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1403185 T5976 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1403188 T5976 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1403189 T5944 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1403189 T5976 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1403190 T5976 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C841 name=collection1 org.apache.solr.core.SolrCore@a4ef1e7 url=http://127.0.0.1:59926/collection1 node=127.0.0.1:59926_ C841_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:59926, core=collection1, node_name=127.0.0.1:59926_}
   [junit4]   2> 1403190 T5976 C:collection1 S:shard1 C841 P59926 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:59926/collection1/
   [junit4]   2> 1403190 T5945 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 1403190 T5976 C:collection1 S:shard1 C841 P59926 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1403190 T5976 C:collection1 S:shard1 C841 P59926 oasc.SyncStrategy.syncToMe http://127.0.0.1:59926/collection1/ has no replicas
   [junit4]   2> 1403191 T5976 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:59926/collection1/ shard1
   [junit4]   2> 1403191 T5976 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1403191 T5971 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1403191 T5944 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1403191 T5956 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1403194 T5944 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1403194 T5945 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59926",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 1403297 T5944 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1403297 T5971 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1403297 T5956 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1403346 T5976 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:59926/collection1/ and leader is http://127.0.0.1:59926/collection1/
   [junit4]   2> 1403347 T5976 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:59926
   [junit4]   2> 1403347 T5976 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1403347 T5976 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1403347 T5976 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1403348 T5944 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1403349 T5945 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59926",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:59926_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1"} current state version: 7
   [junit4]   2> 1403349 T5945 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59926",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:59926_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 1403452 T5956 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1403452 T5944 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1403452 T5971 oascc.ZkStateRe

[...truncated too long message...]

it violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1581182 T6268 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:43674/
   [junit4]   2> 1581182 T6268 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:28515/
   [junit4]   2> 1581183 T6268 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:32427/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=E99E0FB8B730BCED -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=es_VE -Dtests.timezone=WET -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   69.2s J2 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:43674/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([E99E0FB8B730BCED:61CA306219CCD115]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:43674/collection1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:43674 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 54 more
   [junit4]   2> 1581191 T6268 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest E99E0FB8B730BCED-001
   [junit4]   2> 69248 T6267 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {a_t=PostingsFormat(name=LuceneVarGapFixedInterval), multiDefault=PostingsFormat(name=LuceneVarGapFixedInterval), _version_=PostingsFormat(name=Memory doPackFST= false), id=PostingsFormat(name=Memory doPackFST= true), text=PostingsFormat(name=Direct), intDefault=PostingsFormat(name=Memory doPackFST= false), range_facet_l=PostingsFormat(name=LuceneVarGapFixedInterval), timestamp=PostingsFormat(name=LuceneVarGapFixedInterval)}, docValues:{timestamp=DocValuesFormat(name=Direct)}, sim=DefaultSimilarity, locale=es_VE, timezone=WET
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=237957272,total=354942976
   [junit4]   2> NOTE: All tests run in this JVM: [TestFieldTypeResource, CustomCollectionTest, TestHighFrequencyDictionaryFactory, TestSearchPerf, ConvertedLegacyTest, TestManagedSchema, OpenExchangeRatesOrgProviderTest, TestElisionMultitermQuery, DeleteReplicaTest, DistribDocExpirationUpdateProcessorTest, TestCloudInspectUtil, SearchHandlerTest, BasicZkTest, XmlUpdateRequestHandlerTest, DistributedQueryElevationComponentTest, RecoveryZkTest, TestConfigSets, BlockDirectoryTest, TestCollationField, OverseerTest, TestCoreContainer, TestManagedSchemaDynamicFieldResource, TestSurroundQueryParser, TestUtils, TestCopyFieldCollectionResource, ParsingFieldUpdateProcessorsTest, TestDocumentBuilder, TestRecoveryHdfs, AsyncMigrateRouteKeyTest, SolrXmlInZkTest, TestJmxMonitoredMap, HighlighterMaxOffsetTest, ZkNodePropsTest, TestOrdValues, FieldAnalysisRequestHandlerTest, AddSchemaFieldsUpdateProcessorFactoryTest, RequestLoggingTest, TestFunctionQuery, TestStressVersions, TestCryptoKeys, DistanceFunctionTest, MergeStrategyTest, DisMaxRequestHandlerTest, TestFileDictionaryLookup, CursorPagingTest, SortByFunctionTest, UUIDFieldTest, MultiTermTest, TestBinaryField, TestManagedSchemaFieldResource, TestSolrDeletionPolicy2, TestQueryUtils, TestPseudoReturnFields, TestFieldResource, SyncSliceTest, ZkControllerTest, ResponseHeaderTest, TestSolrXml, TestJoin, TestAddFieldRealTimeGet, TestBadConfig, TestReplicationHandler, FullSolrCloudDistribCmdsTest, ReplicationFactorTest, TestBlobHandler, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J2 in 70.03s, 1 test, 1 error <<< FAILURES!

[...truncated 701 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 54 minutes 34 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 231928266 bytes
Compression is 0.0%
Took 46 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2891 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2891/

4 tests failed.
REGRESSION:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
There were too many update fails (31 > 20) - we expect it can happen, but shouldn't easily

Stack Trace:
java.lang.AssertionError: There were too many update fails (31 > 20) - we expect it can happen, but shouldn't easily
	at __randomizedtesting.SeedInfo.seed([59822DFF3468BF56:D1D612259A94D2AE]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertFalse(Assert.java:68)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:230)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:27807/umc/y/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:27807/umc/y/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([59822DFF3468BF56:D1D612259A94D2AE]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:27807 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:25761 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:25761 failed to respond
	at __randomizedtesting.SeedInfo.seed([59822DFF3468BF56:D1D612259A94D2AE]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:25761 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:54581 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:54581 failed to respond
	at __randomizedtesting.SeedInfo.seed([59822DFF3468BF56:D1D612259A94D2AE]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:54581 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9902 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/init-core-data-001
   [junit4]   2> 968972 T2732 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /umc/y
   [junit4]   2> 968977 T2732 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 968978 T2733 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 969078 T2732 oasc.ZkTestServer.run start zk server on port:11023
   [junit4]   2> 969079 T2732 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 969080 T2732 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 969085 T2740 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5524d422 name:ZooKeeperConnection Watcher:127.0.0.1:11023 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 969085 T2732 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 969086 T2732 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 969086 T2732 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 969089 T2732 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 969090 T2732 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 969091 T2743 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1b7f8ea7 name:ZooKeeperConnection Watcher:127.0.0.1:11023/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 969091 T2732 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 969091 T2732 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 969092 T2732 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 969093 T2732 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 969095 T2732 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 969096 T2732 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 969097 T2732 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 969098 T2732 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 969100 T2732 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 969101 T2732 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 969204 T2732 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 969205 T2732 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 969206 T2732 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 969207 T2732 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 969208 T2732 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 969209 T2732 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 969210 T2732 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 969211 T2732 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 969213 T2732 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 969214 T2732 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 969215 T2732 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 969216 T2732 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 969218 T2732 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 969218 T2732 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 969220 T2732 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 969221 T2732 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 969222 T2732 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 969223 T2732 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 970387 T2732 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/cores/collection1
   [junit4]   2> 970390 T2732 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 970392 T2732 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:27802
   [junit4]   2> 970393 T2732 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/cores, hostPort=27801, hostContext=/umc/y}
   [junit4]   2> 970393 T2732 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 970393 T2732 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/'
   [junit4]   2> 970422 T2732 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/solr.xml
   [junit4]   2> 970457 T2732 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/cores
   [junit4]   2> 970458 T2732 oasc.CoreContainer.<init> New CoreContainer 1882061578
   [junit4]   2> 970458 T2732 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/]
   [junit4]   2> 970458 T2732 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/lib
   [junit4]   2> 970459 T2732 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/lib).
   [junit4]   2> 970468 T2732 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 970468 T2732 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 970468 T2732 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 970468 T2732 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 970469 T2732 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 970469 T2732 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 970469 T2732 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 970469 T2732 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 970469 T2732 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 970470 T2732 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 970470 T2732 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 970474 T2732 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 970475 T2732 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 970475 T2732 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 970476 T2732 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 970476 T2732 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:11023/solr
   [junit4]   2> 970476 T2732 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 970476 T2732 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 970477 T2732 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 970478 T2755 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@40b21de1 name:ZooKeeperConnection Watcher:127.0.0.1:11023 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 970479 T2732 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 970479 T2732 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 970480 T2732 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 970481 T2758 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1df671fd name:ZooKeeperConnection Watcher:127.0.0.1:11023/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 970481 T2732 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 970482 T2732 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 970484 T2732 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 970485 T2732 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 970486 T2732 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 970488 T2732 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 970489 T2732 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 970490 T2732 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 970491 T2732 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 970492 T2732 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:27801_umc%2Fy
   [junit4]   2> 970492 T2732 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:27801_umc%2Fy
   [junit4]   2> 970493 T2732 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 970494 T2732 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 970495 T2732 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 970496 T2732 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:27801_umc%2Fy
   [junit4]   2> 970496 T2732 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 970497 T2732 oasc.Overseer.start Overseer (id=93597250280488963-127.0.0.1:27801_umc%2Fy-n_0000000000) starting
   [junit4]   2> 970499 T2732 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 970502 T2732 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 970503 T2760 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 970503 T2732 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 970503 T2759 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 970506 T2732 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/cores
   [junit4]   2> 970508 T2732 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/cores/collection1/, loadOnStartup=true, name=collection1, config=solrconfig.xml, coreNodeName=, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/cores/collection1, collection=control_collection, schema=schema.xml, dataDir=data/}
   [junit4]   2> 970508 T2732 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/cores/collection1/
   [junit4]   2> 970508 T2732 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 970509 T2762 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 970509 T2762 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 970510 T2762 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 970510 T2758 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 970511 T2759 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:27801_umc%2Fy",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27801/umc/y",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null} current state version: 0
   [junit4]   2> 970511 T2759 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:27801_umc%2Fy",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27801/umc/y",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 970511 T2759 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 970511 T2759 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 970512 T2758 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 971510 T2762 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 971511 T2762 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 971511 T2762 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 971512 T2762 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 971512 T2762 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 971513 T2762 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/cores/collection1/'
   [junit4]   2> 971531 T2762 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 971533 T2762 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 971546 T2762 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 971583 T2762 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 971638 T2762 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 971740 T2762 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 971751 T2762 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 972189 T2762 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 972208 T2762 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 972211 T2762 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 972221 T2762 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 972226 T2762 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 972230 T2762 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 972231 T2762 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 972231 T2762 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 972232 T2762 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 972232 T2762 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 972233 T2762 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 972233 T2762 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 972233 T2762 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 972233 T2762 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/cores/collection1/data/
   [junit4]   2> 972234 T2762 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5c2ae567
   [junit4]   2> 972235 T2762 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/cores/collection1/data
   [junit4]   2> 972235 T2762 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/cores/collection1/data/index/
   [junit4]   2> 972235 T2762 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 972236 T2762 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/cores/collection1/data/index
   [junit4]   2> 972236 T2762 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=15, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=15.57421875, floorSegmentMB=0.9921875, forceMergeDeletesPctAllowed=2.274594146939587, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 972237 T2762 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@68362e4f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3d1a79fc),segFN=segments_1,generation=1}
   [junit4]   2> 972237 T2762 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 972243 T2762 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 972243 T2762 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 972243 T2762 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 972244 T2762 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 972244 T2762 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 972244 T2762 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 972245 T2762 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 972245 T2762 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 972245 T2762 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 972273 T2762 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 972275 T2762 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 972278 T2762 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 972280 T2762 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 972286 T2762 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /get,/admin/file,/schema,/admin/luke,/admin/plugins,/admin/threads,/admin/segments,/admin/mbeans,/update,/replication,/admin/ping,/update/json,/update/json/docs,/update/csv,/admin/logging,standard,/admin/system,/config,/admin/properties
   [junit4]   2> 972288 T2762 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 972290 T2762 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 972290 T2762 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 972292 T2762 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 972292 T2762 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 972293 T2762 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=29, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=86.19140625, floorSegmentMB=0.2578125, forceMergeDeletesPctAllowed=11.354393203908225, segmentsPerTier=45.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 972293 T2762 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@68362e4f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3d1a79fc),segFN=segments_1,generation=1}
   [junit4]   2> 972294 T2762 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 972294 T2762 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@53c4f39f[collection1] main
   [junit4]   2> 972294 T2762 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 972295 T2762 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 972295 T2762 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 972296 T2762 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 972296 T2762 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 972296 T2762 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 972297 T2762 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 972297 T2762 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 972297 T2762 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 972298 T2762 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 972299 T2763 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@53c4f39f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 972299 T2762 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 972300 T2766 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:27801/umc/y collection:control_collection shard:shard1
   [junit4]   2> 972300 T2732 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 972301 T2732 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 972301 T2766 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 972304 T2766 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 972306 T2758 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 972306 T2766 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 972306 T2732 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 972306 T2759 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 972306 T2766 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C331 name=collection1 org.apache.solr.core.SolrCore@5cff8658 url=http://127.0.0.1:27801/umc/y/collection1 node=127.0.0.1:27801_umc%2Fy C331_STATE=coll:control_collection core:collection1 props:{state=down, node_name=127.0.0.1:27801_umc%2Fy, core=collection1, base_url=http://127.0.0.1:27801/umc/y}
   [junit4]   2> 972307 T2766 C:control_collection S:shard1 C331 P27801 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:27801/umc/y/collection1/
   [junit4]   2> 972307 T2732 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 972307 T2766 C:control_collection S:shard1 C331 P27801 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 972308 T2766 C:control_collection S:shard1 C331 P27801 oasc.SyncStrategy.syncToMe http://127.0.0.1:27801/umc/y/collection1/ has no replicas
   [junit4]   2> 972308 T2766 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:27801/umc/y/collection1/ shard1
   [junit4]   2> 972308 T2766 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 972308 T2770 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5418b778 name:ZooKeeperConnection Watcher:127.0.0.1:11023/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 972308 T2758 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 972309 T2732 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 972309 T2732 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 972310 T2732 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 972311 T2758 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 972312 T2759 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27801/umc/y",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 972312 T2732 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 972416 T2770 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 972416 T2758 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 972465 T2766 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:27801/umc/y/collection1/ and leader is http://127.0.0.1:27801/umc/y/collection1/
   [junit4]   2> 972466 T2766 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:27801/umc/y
   [junit4]   2> 972466 T2766 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 972466 T2766 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 972466 T2766 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 972467 T2758 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 972468 T2759 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:27801_umc%2Fy",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27801/umc/y",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1"} current state version: 3
   [junit4]   2> 972468 T2759 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:27801_umc%2Fy",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27801/umc/y",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 972571 T2758 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 972571 T2770 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 973746 T2732 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/cores/collection1
   [junit4]   2> 973749 T2732 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001
   [junit4]   2> 973749 T2732 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 973752 T2732 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:27808
   [junit4]   2> 973752 T2732 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/tempDir-001/jetty1, hostContext=/umc/y, hostPort=27807, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/cores, solrconfig=solrconfig.xml}
   [junit4]   2> 973753 T2732 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 973753 T2732 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/'
   [junit4]   2> 973790 T2732 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/solr.xml
   [junit4]   2> 973857 T2732 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/cores
   [junit4]   2> 973857 T2732 oasc.CoreContainer.<init> New CoreContainer 326067208
   [junit4]   2> 973858 T2732 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/]
   [junit4]   2> 973858 T2732 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/lib
   [junit4]   2> 973859 T2732 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/lib).
   [junit4]   2> 973873 T2732 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 973874 T2732 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 973874 T2732 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 973874 T2732 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 973875 T2732 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 973875 T2732 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 973875 T2732 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 973876 T2732 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 973876 T2732 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 973876 T2732 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 973877 T2732 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 973887 T2732 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 973888 T2732 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 973888 T2732 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 973889 T2732 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 973889 T2732 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:11023/solr
   [junit4]   2> 973889 T2732 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 973890 T2732 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 973891 T2732 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 973892 T2782 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@64dccefe name:ZooKeeperConnection Watcher:127.0.0.1:11023 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 973893 T2732 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 973894 T2732 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 973896 T2732 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 973897 T2785 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3ea6b446 name:ZooKeeperConnection Watcher:127.0.0.1:11023/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 973897 T2732 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 973901 T2732 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 974906 T2732 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:27807_umc%2Fy
   [junit4]   2> 974907 T2732 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:27807_umc%2Fy
   [junit4]   2> 974910 T2732 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 974912 T2732 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/cores
   [junit4]   2> 974914 T2732 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/cores/collection1, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/cores/collection1/, transient=false, loadOnStartup=true, coreNodeName=, collection=collection1, shard=, name=collection1, dataDir=data/, schema=schema.xml}
   [junit4]   2> 974914 T2732 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/cores/collection1/
   [junit4]   2> 974914 T2732 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 974915 T2786 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 974915 T2786 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 974916 T2758 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 974916 T2786 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 974917 T2759 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:27807_umc%2Fy",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27807/umc/y",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null} current state version: 4
   [junit4]   2> 974917 T2759 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:27807_umc%2Fy",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27807/umc/y",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 974917 T2759 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 974918 T2759 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 974919 T2770 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 974919 T2785 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 974919 T2758 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 975916 T2786 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 975917 T2786 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 975917 T2786 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 975918 T2786 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 975918 T2786 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 975918 T2786 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/cores/collection1/'
   [junit4]   2> 975931 T2786 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 975933 T2786 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 975940 T2786 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 975956 T2786 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 975989 T2786 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 976091 T2786 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 976098 T2786 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 976387 T2786 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 976401 T2786 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 976403 T2786 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 976410 T2786 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 976414 T2786 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 976417 T2786 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 976417 T2786 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 976417 T2786 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 976418 T2786 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 976418 T2786 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 976418 T2786 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 976418 T2786 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 976419 T2786 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 976419 T2786 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 976419 T2786 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5c2ae567
   [junit4]   2> 976420 T2786 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/cores/collection1/data
   [junit4]   2> 976420 T2786 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 976420 T2786 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 976420 T2786 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 976421 T2786 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=15, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=15.57421875, floorSegmentMB=0.9921875, forceMergeDeletesPctAllowed=2.274594146939587, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 976421 T2786 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2ba8d1c0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5862aaa4),segFN=segments_1,generation=1}
   [junit4]   2> 976421 T2786 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 976426 T2786 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 976426 T2786 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 976427 T2786 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 976427 T2786 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 976427 T2786 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 976427 T2786 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 976428 T2786 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 976428 T2786 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 976428 T2786 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 976450 T2786 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 976452 T2786 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 976454 T2786 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 976456 T2786 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 976461 T2786 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /get,/admin/file,/schema,/admin/luke,/admin/plugins,/admin/threads,/admin/segments,/admin/mbeans,/update,/replication,/admin/ping,/update/json,/update/json/docs,/update/csv,/admin/logging,standard,/admin/system,/config,/admin/properties
   [junit4]   2> 976463 T2786 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 976464 T2786 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 976464 T2786 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 976466 T2786 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 976466 T2786 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 976467 T2786 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=29, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=86.19140625, floorSegmentMB=0.2578125, forceMergeDeletesPctAllowed=11.354393203908225, segmentsPerTier=45.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 976468 T2786 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2ba8d1c0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5862aaa4),segFN=segments_1,generation=1}
   [junit4]   2> 976468 T2786 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 976468 T2786 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@6fe92c91[collection1] main
   [junit4]   2> 976469 T2786 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 976469 T2786 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 976470 T2786 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 976470 T2786 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 976470 T2786 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 976470 T2786 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 976471 T2786 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 976471 T2786 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 976471 T2786 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 976471 T2786 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 976472 T2787 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6fe92c91[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 976473 T2786 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 976473 T2790 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:27807/umc/y collection:collection1 shard:shard1
   [junit4]   2> 976474 T2732 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 976474 T2790 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 976474 T2732 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 976477 T2790 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 976478 T2758 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 976478 T2790 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 976478 T2790 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C332 name=collection1 org.apache.solr.core.SolrCore@1bc9848f url=http://127.0.0.1:27807/umc/y/collection1 node=127.0.0.1:27807_umc%2Fy C332_STATE=coll:collection1 core:collection1 props:{state=down, node_name=127.0.0.1:27807_umc%2Fy, core=collection1, base_url=http://127.0.0.1:27807/umc/y}
   [junit4]   2> 976478 T2790 C:collection1 S:shard1 C332 P27807 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:27807/umc/y/collection1/
   [junit4]   2> 976479 T2759 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 976479 T2790 C:collection1 S:shard1 C332 P27807 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 976479 T2790 C:collection1 S:shard1 C332 P27807 oasc.SyncStrategy.syncToMe http://127.0.0.1:27807/umc/y/collection1/ has no rep

[...truncated too long message...]

y: stop shard! 29705
   [junit4]   2> 1582636 T3248 C:control_collection S:shard1 c:collection1 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=192973817
   [junit4]   2> 1582636 T3248 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1582636 T3248 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1582638 T3409 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1582639 T3248 C:control_collection S:shard1 c:collection1 oasc.Overseer.close Overseer (id=93597279702548490-127.0.0.1:29705_qiue%2Fi-n_0000000003) closing
   [junit4]   2> 1582639 T3411 oasc.Overseer$ClusterStateUpdater.run Overseer Loop exiting : 127.0.0.1:29705_qiue%2Fi
   [junit4]   2> 1582641 T3409 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1583445 T3402 C457 P29705 oasc.SyncStrategy.sync WARN Closed, skipping sync up.
   [junit4]   2> 1583447 T3402 oasc.ShardLeaderElectionContext.rejoinLeaderElection Not rejoining election because CoreContainer is closed
   [junit4]   2> 1583447 T3402 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@1957e1b4
   [junit4]   2> 1583447 T3402 oasc.ZkController.unRegisterConfListener  a listener was removed because of core close
   [junit4]   2> 1583448 T3402 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=2,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=501,cumulative_deletesById=241,cumulative_deletesByQuery=1,cumulative_errors=0,transaction_logs_total_size=73417,transaction_logs_total_number=1}
   [junit4]   2> 1583449 T3402 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 1583450 T3402 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 1583450 T3402 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 1583452 T3402 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 1583471 T3402 oasc.CachingDirectoryFactory.close Closing StandardDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 1583471 T3402 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 59822DFF3468BF56-001/shard-3-001/cores/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 59822DFF3468BF56-001/shard-3-001/cores/collection1/data;done=false>>]
   [junit4]   2> 1583471 T3402 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 59822DFF3468BF56-001/shard-3-001/cores/collection1/data
   [junit4]   2> 1583472 T3402 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 59822DFF3468BF56-001/shard-3-001/cores/collection1/data/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 59822DFF3468BF56-001/shard-3-001/cores/collection1/data/index;done=false>>]
   [junit4]   2> 1583472 T3402 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 59822DFF3468BF56-001/shard-3-001/cores/collection1/data/index
   [junit4]   2> 1583474 T3248 C:control_collection S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/qiue/i,null}
   [junit4]   2> 1583677 T3248 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:26120 26120
   [junit4]   2> 1583737 T3377 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:26120 26120
   [junit4]   2> 1583741 T3377 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		2	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ChaosMonkeyNothingIsSafeTest -Dtests.method=test -Dtests.seed=59822DFF3468BF56 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=sv_SE -Dtests.timezone=America/Argentina/Buenos_Aires -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE  166s J1 | ChaosMonkeyNothingIsSafeTest.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: There were too many update fails (31 > 20) - we expect it can happen, but shouldn't easily
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([59822DFF3468BF56:D1D612259A94D2AE]:0)
   [junit4]    > 	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:230)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1583750 T3248 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 59822DFF3468BF56-001
   [junit4]   2> 165957 T3247 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {a_i=PostingsFormat(name=Direct), _version_=FST50, id=PostingsFormat(name=Direct), a_t=FST50, rnd_b=PostingsFormat(name=Direct)}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=sv_SE, timezone=America/Argentina/Buenos_Aires
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=75621816,total=241172480
   [junit4]   2> NOTE: All tests run in this JVM: [TestLMDirichletSimilarityFactory, CoreMergeIndexesAdminHandlerTest, SolrTestCaseJ4Test, HdfsWriteToMultipleCollectionsTest, TestExpandComponent, TestFieldTypeCollectionResource, DistributedFacetPivotSmallAdvancedTest, TestRTGBase, TestSolrIndexConfig, TestWriterPerf, SimplePostToolTest, TestLuceneMatchVersion, TestCoreDiscovery, TestLFUCache, HdfsLockFactoryTest, TestSimpleTrackingShardHandler, TestReversedWildcardFilterFactory, RequestHandlersTest, CacheHeaderTest, DeleteShardTest, TestLMJelinekMercerSimilarityFactory, AtomicUpdatesTest, SolrXmlInZkTest, OverseerTest, NumericFieldsTest, CollectionsAPIAsyncDistributedZkTest, RollingRestartTest, ClusterStateTest, TestCursorMarkWithoutUniqueKey, TestShortCircuitedRequests, TestBadConfig, SolrCmdDistributorTest, BadCopyFieldTest, TestAnalyzeInfixSuggestions, FacetPivotSmallTest, LeaderFailoverAfterPartitionTest, TestFastWriter, OverseerRolesTest, TestCSVResponseWriter, TestRecoveryHdfs, AsyncMigrateRouteKeyTest, TestCloudManagedSchema, ClusterStateUpdateTest, TestStandardQParsers, HdfsCollectionsAPIDistributedZkTest, SolrIndexConfigTest, ShardSplitTest, TestSolrCoreProperties, CircularListTest, TestCSVLoader, TestManagedSchema, DistributedFacetPivotWhiteBoxTest, PreAnalyzedUpdateProcessorTest, TestMiniSolrCloudClusterSSL, RecoveryAfterSoftCommitTest, TestFaceting, ChaosMonkeyNothingIsSafeTest]
   [junit4] Completed on J1 in 166.51s, 1 test, 1 failure <<< FAILURES!

[...truncated 702 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 1 failure, 61 ignored (21 assumptions)

Total time: 54 minutes 51 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 210106413 bytes
Compression is 0.0%
Took 32 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2890 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2890/

4 tests failed.
REGRESSION:  org.apache.solr.cloud.TestCloudPivotFacet.test

Error Message:
{main(facet=true&facet.pivot=%7B%21stats%3Dst3%7Dpivot_ti%2Cpivot_tl%2Cpivot_dt&facet.limit=16&facet.offset=3&facet.pivot.mincount=3&facet.missing=true&facet.sort=index&facet.overrequest.ratio=-1.1108205),extra(rows=0&q=*%3A*&fq=id%3A%5B*+TO+665%5D&stats=true&stats.field=%7B%21key%3Dsk1+tag%3Dst1%2Cst2%7Dpivot_tdt1&stats.field=%7B%21key%3Dsk2+tag%3Dst2%2Cst3%7Dpivot_ti&stats.field=%7B%21key%3Dsk3+tag%3Dst3%2Cst4%7Dpivot_tl1&_test_min=3&_test_miss=true&_test_sort=index)} ==> pivot_ti,pivot_tl,pivot_dt: {params(rows=0),defaults({main({main(rows=0&q=*%3A*&fq=id%3A%5B*+TO+665%5D&stats=true&stats.field=%7B%21key%3Dsk1+tag%3Dst1%2Cst2%7Dpivot_tdt1&stats.field=%7B%21key%3Dsk2+tag%3Dst2%2Cst3%7Dpivot_ti&stats.field=%7B%21key%3Dsk3+tag%3Dst3%2Cst4%7Dpivot_tl1&_test_min=3&_test_miss=true&_test_sort=index),extra(fq=%7B%21term+f%3Dpivot_ti%7D35)}),extra(fq=-pivot_tl%3A%5B*+TO+*%5D)})} expected:<4> but was:<5>

Stack Trace:
java.lang.AssertionError: {main(facet=true&facet.pivot=%7B%21stats%3Dst3%7Dpivot_ti%2Cpivot_tl%2Cpivot_dt&facet.limit=16&facet.offset=3&facet.pivot.mincount=3&facet.missing=true&facet.sort=index&facet.overrequest.ratio=-1.1108205),extra(rows=0&q=*%3A*&fq=id%3A%5B*+TO+665%5D&stats=true&stats.field=%7B%21key%3Dsk1+tag%3Dst1%2Cst2%7Dpivot_tdt1&stats.field=%7B%21key%3Dsk2+tag%3Dst2%2Cst3%7Dpivot_ti&stats.field=%7B%21key%3Dsk3+tag%3Dst3%2Cst4%7Dpivot_tl1&_test_min=3&_test_miss=true&_test_sort=index)} ==> pivot_ti,pivot_tl,pivot_dt: {params(rows=0),defaults({main({main(rows=0&q=*%3A*&fq=id%3A%5B*+TO+665%5D&stats=true&stats.field=%7B%21key%3Dsk1+tag%3Dst1%2Cst2%7Dpivot_tdt1&stats.field=%7B%21key%3Dsk2+tag%3Dst2%2Cst3%7Dpivot_ti&stats.field=%7B%21key%3Dsk3+tag%3Dst3%2Cst4%7Dpivot_tl1&_test_min=3&_test_miss=true&_test_sort=index),extra(fq=%7B%21term+f%3Dpivot_ti%7D35)}),extra(fq=-pivot_tl%3A%5B*+TO+*%5D)})} expected:<4> but was:<5>
	at __randomizedtesting.SeedInfo.seed([12A3AA716553D370:9AF795ABCBAFBE88]:0)
	at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotCountsAreCorrect(TestCloudPivotFacet.java:281)
	at org.apache.solr.cloud.TestCloudPivotFacet.test(TestCloudPivotFacet.java:228)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.AssertionError: pivot_ti,pivot_tl,pivot_dt: {params(rows=0),defaults({main({main(rows=0&q=*%3A*&fq=id%3A%5B*+TO+665%5D&stats=true&stats.field=%7B%21key%3Dsk1+tag%3Dst1%2Cst2%7Dpivot_tdt1&stats.field=%7B%21key%3Dsk2+tag%3Dst2%2Cst3%7Dpivot_ti&stats.field=%7B%21key%3Dsk3+tag%3Dst3%2Cst4%7Dpivot_tl1&_test_min=3&_test_miss=true&_test_sort=index),extra(fq=%7B%21term+f%3Dpivot_ti%7D35)}),extra(fq=-pivot_tl%3A%5B*+TO+*%5D)})} expected:<4> but was:<5>
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.apache.solr.cloud.TestCloudPivotFacet.assertNumFound(TestCloudPivotFacet.java:677)
	at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotData(TestCloudPivotFacet.java:335)
	at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotCountsAreCorrect(TestCloudPivotFacet.java:302)
	at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotCountsAreCorrect(TestCloudPivotFacet.java:313)
	at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotCountsAreCorrect(TestCloudPivotFacet.java:271)
	... 42 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:13551/lv_/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:13551/lv_/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([12A3AA716553D370:9AF795ABCBAFBE88]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:13551 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:13793/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:13793/collection1
	at __randomizedtesting.SeedInfo.seed([12A3AA716553D370:9AF795ABCBAFBE88]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:13793/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:13793 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:58620/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:58620/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([12A3AA716553D370:9AF795ABCBAFBE88]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:58620/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:58620 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9791 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/init-core-data-001
   [junit4]   2> 669602 T2624 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 669607 T2624 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 669607 T2625 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 669707 T2624 oasc.ZkTestServer.run start zk server on port:58617
   [junit4]   2> 669708 T2624 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 669709 T2624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 669712 T2632 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7172a99e name:ZooKeeperConnection Watcher:127.0.0.1:58617 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 669712 T2624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 669713 T2624 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 669713 T2624 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 669715 T2624 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 669716 T2624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 669717 T2635 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4575d7f2 name:ZooKeeperConnection Watcher:127.0.0.1:58617/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 669717 T2624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 669718 T2624 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 669718 T2624 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 669720 T2624 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 669721 T2624 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 669722 T2624 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 669724 T2624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 669724 T2624 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 669726 T2624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 669726 T2624 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 669829 T2624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 669829 T2624 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 669831 T2624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 669831 T2624 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 669832 T2624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 669833 T2624 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 669834 T2624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 669834 T2624 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 669836 T2624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 669836 T2624 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 669837 T2624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 669838 T2624 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 669839 T2624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 669839 T2624 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 669841 T2624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 669841 T2624 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 669842 T2624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 669843 T2624 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 670822 T2624 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/cores/collection1
   [junit4]   2> 670826 T2624 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 670828 T2624 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:58621
   [junit4]   2> 670829 T2624 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/tempDir-001/control/data, hostPort=58620, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/cores}
   [junit4]   2> 670829 T2624 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 670830 T2624 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/'
   [junit4]   2> 670863 T2624 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/solr.xml
   [junit4]   2> 670905 T2624 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/cores
   [junit4]   2> 670905 T2624 oasc.CoreContainer.<init> New CoreContainer 54629337
   [junit4]   2> 670905 T2624 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/]
   [junit4]   2> 670906 T2624 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/lib
   [junit4]   2> 670906 T2624 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/lib).
   [junit4]   2> 670916 T2624 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 670917 T2624 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 670917 T2624 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 670917 T2624 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 670917 T2624 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 670917 T2624 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 670918 T2624 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 670918 T2624 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 670918 T2624 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 670918 T2624 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 670919 T2624 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 670924 T2624 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 670924 T2624 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 670925 T2624 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 670925 T2624 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 670925 T2624 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:58617/solr
   [junit4]   2> 670925 T2624 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 670926 T2624 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 670926 T2624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 670928 T2647 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4b038a52 name:ZooKeeperConnection Watcher:127.0.0.1:58617 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 670929 T2624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 670929 T2624 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 670931 T2624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 670933 T2650 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@aa6d68c name:ZooKeeperConnection Watcher:127.0.0.1:58617/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 670933 T2624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 670935 T2624 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 670937 T2624 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 670938 T2624 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 670940 T2624 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 670942 T2624 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 670943 T2624 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 670945 T2624 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 670946 T2624 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 670947 T2624 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58620_
   [junit4]   2> 670947 T2624 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:58620_
   [junit4]   2> 670948 T2624 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 670950 T2624 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 670951 T2624 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 670952 T2624 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:58620_
   [junit4]   2> 670952 T2624 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 670953 T2624 oasc.Overseer.start Overseer (id=93596374343286787-127.0.0.1:58620_-n_0000000000) starting
   [junit4]   2> 670955 T2624 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 670960 T2624 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 670960 T2652 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 670960 T2624 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 670961 T2651 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 670965 T2624 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/cores
   [junit4]   2> 670967 T2624 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=control_collection, loadOnStartup=true, shard=, schema=schema.xml, name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/cores/collection1/, dataDir=data/, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/cores/collection1, config=solrconfig.xml, transient=false}
   [junit4]   2> 670967 T2624 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/cores/collection1/
   [junit4]   2> 670967 T2624 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 670968 T2654 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 670968 T2654 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 670969 T2654 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 670969 T2650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 670970 T2651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:58620_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58620",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 0
   [junit4]   2> 670971 T2651 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:58620_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58620",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 670971 T2651 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 670971 T2651 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 670972 T2650 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 671970 T2654 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 671971 T2654 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 671971 T2654 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 671971 T2654 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 671972 T2654 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 671972 T2654 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/cores/collection1/'
   [junit4]   2> 671986 T2654 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 671988 T2654 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 671997 T2654 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 672015 T2654 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 672066 T2654 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 672167 T2654 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 672175 T2654 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 672491 T2654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 672504 T2654 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 672506 T2654 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 672514 T2654 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 672518 T2654 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 672521 T2654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 672522 T2654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 672522 T2654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 672522 T2654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 672523 T2654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 672523 T2654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 672523 T2654 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 672523 T2654 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 672523 T2654 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/cores/collection1/data/
   [junit4]   2> 672524 T2654 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e6cd229
   [junit4]   2> 672524 T2654 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/cores/collection1/data
   [junit4]   2> 672525 T2654 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/cores/collection1/data/index/
   [junit4]   2> 672525 T2654 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 672525 T2654 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/cores/collection1/data/index
   [junit4]   2> 672525 T2654 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=42, maxMergeAtOnceExplicit=46, maxMergedSegmentMB=63.03515625, floorSegmentMB=1.435546875, forceMergeDeletesPctAllowed=5.447954420730018, segmentsPerTier=23.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 672526 T2654 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3c4b7168 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7d0281c4),segFN=segments_1,generation=1}
   [junit4]   2> 672526 T2654 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 672530 T2654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 672530 T2654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 672531 T2654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 672531 T2654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 672531 T2654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 672531 T2654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 672532 T2654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 672532 T2654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 672532 T2654 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 672549 T2654 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 672552 T2654 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 672553 T2654 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 672555 T2654 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 672559 T2654 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/logging,/replication,/schema,/admin/segments,/admin/luke,/update/json/docs,/get,/admin/file,/update/json,/admin/properties,/admin/system,standard,/admin/plugins,/admin/mbeans,/admin/ping,/admin/threads,/update,/config,/update/csv
   [junit4]   2> 672561 T2654 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 672562 T2654 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 672562 T2654 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 672564 T2654 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 672564 T2654 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 672565 T2654 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=17, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=39.8115234375, floorSegmentMB=0.7158203125, forceMergeDeletesPctAllowed=2.9634851000593088, segmentsPerTier=50.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 672565 T2654 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3c4b7168 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7d0281c4),segFN=segments_1,generation=1}
   [junit4]   2> 672565 T2654 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 672566 T2654 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@77cf9b72[collection1] main
   [junit4]   2> 672566 T2654 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 672567 T2654 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 672567 T2654 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 672567 T2654 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 672567 T2654 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 672567 T2654 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 672568 T2654 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 672568 T2654 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 672568 T2654 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 672568 T2654 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 672569 T2655 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@77cf9b72[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 672570 T2654 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 672570 T2658 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:58620 collection:control_collection shard:shard1
   [junit4]   2> 672570 T2624 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 672571 T2624 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 672571 T2658 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 672574 T2658 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 672574 T2624 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 672575 T2624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 672575 T2650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 672576 T2658 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 672576 T2662 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@49b0d9d1 name:ZooKeeperConnection Watcher:127.0.0.1:58617/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 672576 T2651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 672576 T2658 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 672576 T2624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> ASYNC  NEW_CORE C361 name=collection1 org.apache.solr.core.SolrCore@801aa0a url=http://127.0.0.1:58620/collection1 node=127.0.0.1:58620_ C361_STATE=coll:control_collection core:collection1 props:{state=down, node_name=127.0.0.1:58620_, core=collection1, base_url=http://127.0.0.1:58620}
   [junit4]   2> 672577 T2658 C:control_collection S:shard1 C361 P58620 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:58620/collection1/
   [junit4]   2> 672577 T2624 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 672577 T2658 C:control_collection S:shard1 C361 P58620 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 672577 T2650 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 672577 T2624 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 672577 T2658 C:control_collection S:shard1 C361 P58620 oasc.SyncStrategy.syncToMe http://127.0.0.1:58620/collection1/ has no replicas
   [junit4]   2> 672579 T2658 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:58620/collection1/ shard1
   [junit4]   2> 672579 T2658 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 672581 T2624 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 672581 T2624 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 672581 T2624 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 672582 T2650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 672582 T2624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 672583 T2651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58620",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 672583 T2665 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@38586f43 name:ZooKeeperConnection Watcher:127.0.0.1:58617/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 672583 T2624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 672584 T2624 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 672586 T2650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 672587 T2651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 672587 T2651 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 672588 T2650 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 672588 T2662 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 672633 T2658 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:58620/collection1/ and leader is http://127.0.0.1:58620/collection1/
   [junit4]   2> 672634 T2658 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:58620
   [junit4]   2> 672634 T2658 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 672634 T2658 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 672634 T2658 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 672635 T2650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 672636 T2651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:58620_",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58620",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 3
   [junit4]   2> 672637 T2651 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:58620_",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58620",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 672637 T2651 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 672638 T2650 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 672638 T2662 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 672742 T2662 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 672742 T2650 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 673545 T2624 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/cores/collection1
   [junit4]   2> 673547 T2624 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001
   [junit4]   2> 673547 T2624 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 673550 T2624 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:58631
   [junit4]   2> 673550 T2624 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solrconfig=solrconfig.xml, hostPort=58630, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/tempDir-001/jetty1, hostContext=/}
   [junit4]   2> 673551 T2624 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 673551 T2624 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/'
   [junit4]   2> 673581 T2624 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/solr.xml
   [junit4]   2> 673616 T2624 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/cores
   [junit4]   2> 673617 T2624 oasc.CoreContainer.<init> New CoreContainer 592846894
   [junit4]   2> 673617 T2624 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/]
   [junit4]   2> 673617 T2624 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/lib
   [junit4]   2> 673618 T2624 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/lib).
   [junit4]   2> 673627 T2624 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 673627 T2624 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 673628 T2624 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 673628 T2624 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 673628 T2624 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 673628 T2624 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 673628 T2624 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 673629 T2624 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 673629 T2624 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 673629 T2624 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 673629 T2624 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 673634 T2624 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 673635 T2624 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 673635 T2624 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 673635 T2624 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 673635 T2624 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:58617/solr
   [junit4]   2> 673636 T2624 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 673636 T2624 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 673636 T2624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 673638 T2677 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@62a364f name:ZooKeeperConnection Watcher:127.0.0.1:58617 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 673639 T2624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 673639 T2624 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 673641 T2624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 673643 T2680 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@74e7039a name:ZooKeeperConnection Watcher:127.0.0.1:58617/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 673643 T2624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 673646 T2624 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 674651 T2624 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58630_
   [junit4]   2> 674652 T2624 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:58630_
   [junit4]   2> 674655 T2624 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 674657 T2624 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/cores
   [junit4]   2> 674659 T2624 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/cores/collection1, transient=false, config=solrconfig.xml, schema=schema.xml, collection=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/cores/collection1/, loadOnStartup=true, name=collection1, coreNodeName=, dataDir=data/}
   [junit4]   2> 674660 T2624 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/cores/collection1/
   [junit4]   2> 674660 T2624 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 674662 T2681 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 674662 T2681 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 674663 T2650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 674664 T2681 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 674664 T2681 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 674664 T2651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:58630_",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58630",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 674664 T2681 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 674665 T2651 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:58630_",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58630",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1"}
   [junit4]   2> 674665 T2651 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 674665 T2651 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 674665 T2681 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 674665 T2681 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 674666 T2651 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 674667 T2680 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 674667 T2680 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 675666 T2681 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 675667 T2681 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 675667 T2681 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 675668 T2681 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 675668 T2681 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 675668 T2681 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/cores/collection1/'
   [junit4]   2> 675681 T2681 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 675683 T2681 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 675691 T2681 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 675707 T2681 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 675742 T2681 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 675843 T2681 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 675850 T2681 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 676167 T2681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 676181 T2681 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 676183 T2681 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 676191 T2681 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 676195 T2681 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 676198 T2681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 676199 T2681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 676199 T2681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 676199 T2681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 676200 T2681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 676200 T2681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 676200 T2681 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 676200 T2681 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 676201 T2681 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 676201 T2681 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e6cd229
   [junit4]   2> 676201 T2681 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/cores/collection1/data
   [junit4]   2> 676202 T2681 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 676202 T2681 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 676202 T2681 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 676203 T2681 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=42, maxMergeAtOnceExplicit=46, maxMergedSegmentMB=63.03515625, floorSegmentMB=1.435546875, forceMergeDeletesPctAllowed=5.447954420730018, segmentsPerTier=23.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 676203 T2681 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5043cc65 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1320240e),segFN=segments_1,generation=1}
   [junit4]   2> 676203 T2681 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 676208 T2681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 676209 T2681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 676209 T2681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 676209 T2681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 676209 T2681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 676209 T2681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 676210 T2681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 676210 T2681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 676210 T2681 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 676236 T2681 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 676239 T2681 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 676240 T2681 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 676242 T2681 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 676248 T2681 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/logging,/replication,/schema,/admin/segments,/admin/luke,/update/json/docs,/get,/admin/file,/update/json,/admin/properties,/admin/system,standard,/admin/plugins,/admin/mbeans,/admin/ping,/admin/threads,/update,/config,/update/csv
   [junit4]   2> 676250 T2681 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 676251 T2681 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 676251 T2681 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 676252 T2681 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 676252 T2681 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 676253 T2681 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=17, maxMergeAt

[...truncated too long message...]

;done=false>>]
   [junit4]   2> 1643957 T5295 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet 12A3AA716553D370-001/shard-4-001/cores/collection1/data/index
   [junit4]   2> 1643959 T5131 C:control_collection S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/_mlt/rs,null}
   [junit4]   2> 1644163 T5131 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:58148 58148
   [junit4]   2> 1644230 T5132 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:58148 58148
   [junit4]   2> 1644232 T5132 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		4	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/queue
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestCloudPivotFacet -Dtests.method=test -Dtests.seed=12A3AA716553D370 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=vi -Dtests.timezone=Etc/GMT-11 -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] FAILURE 52.3s J1 | TestCloudPivotFacet.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: {main(facet=true&facet.pivot=%7B%21stats%3Dst3%7Dpivot_ti%2Cpivot_tl%2Cpivot_dt&facet.limit=16&facet.offset=3&facet.pivot.mincount=3&facet.missing=true&facet.sort=index&facet.overrequest.ratio=-1.1108205),extra(rows=0&q=*%3A*&fq=id%3A%5B*+TO+665%5D&stats=true&stats.field=%7B%21key%3Dsk1+tag%3Dst1%2Cst2%7Dpivot_tdt1&stats.field=%7B%21key%3Dsk2+tag%3Dst2%2Cst3%7Dpivot_ti&stats.field=%7B%21key%3Dsk3+tag%3Dst3%2Cst4%7Dpivot_tl1&_test_min=3&_test_miss=true&_test_sort=index)} ==> pivot_ti,pivot_tl,pivot_dt: {params(rows=0),defaults({main({main(rows=0&q=*%3A*&fq=id%3A%5B*+TO+665%5D&stats=true&stats.field=%7B%21key%3Dsk1+tag%3Dst1%2Cst2%7Dpivot_tdt1&stats.field=%7B%21key%3Dsk2+tag%3Dst2%2Cst3%7Dpivot_ti&stats.field=%7B%21key%3Dsk3+tag%3Dst3%2Cst4%7Dpivot_tl1&_test_min=3&_test_miss=true&_test_sort=index),extra(fq=%7B%21term+f%3Dpivot_ti%7D35)}),extra(fq=-pivot_tl%3A%5B*+TO+*%5D)})} expected:<4> but was:<5>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([12A3AA716553D370:9AF795ABCBAFBE88]:0)
   [junit4]    > 	at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotCountsAreCorrect(TestCloudPivotFacet.java:281)
   [junit4]    > 	at org.apache.solr.cloud.TestCloudPivotFacet.test(TestCloudPivotFacet.java:228)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: java.lang.AssertionError: pivot_ti,pivot_tl,pivot_dt: {params(rows=0),defaults({main({main(rows=0&q=*%3A*&fq=id%3A%5B*+TO+665%5D&stats=true&stats.field=%7B%21key%3Dsk1+tag%3Dst1%2Cst2%7Dpivot_tdt1&stats.field=%7B%21key%3Dsk2+tag%3Dst2%2Cst3%7Dpivot_ti&stats.field=%7B%21key%3Dsk3+tag%3Dst3%2Cst4%7Dpivot_tl1&_test_min=3&_test_miss=true&_test_sort=index),extra(fq=%7B%21term+f%3Dpivot_ti%7D35)}),extra(fq=-pivot_tl%3A%5B*+TO+*%5D)})} expected:<4> but was:<5>
   [junit4]    > 	at org.apache.solr.cloud.TestCloudPivotFacet.assertNumFound(TestCloudPivotFacet.java:677)
   [junit4]    > 	at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotData(TestCloudPivotFacet.java:335)
   [junit4]    > 	at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotCountsAreCorrect(TestCloudPivotFacet.java:302)
   [junit4]    > 	at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotCountsAreCorrect(TestCloudPivotFacet.java:313)
   [junit4]    > 	at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotCountsAreCorrect(TestCloudPivotFacet.java:271)
   [junit4]    > 	... 42 more
   [junit4]   2> 1644239 T5131 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet 12A3AA716553D370-001
   [junit4]   2> 52321 T5130 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {pivot_ti=BlockTreeOrds(blocksize=128), pivot_b=FSTOrd50, pivot_z_s1=BlockTreeOrds(blocksize=128), pivot_tf1=BlockTreeOrds(blocksize=128), pivot_tdt1=PostingsFormat(name=Direct), pivot_tl1=PostingsFormat(name=Direct), pivot_z_s=FSTOrd50, pivot_f1=PostingsFormat(name=Direct), range_facet_l=PostingsFormat(name=Direct), id=Lucene50(blocksize=128), dense_pivot_y_s=BlockTreeOrds(blocksize=128), pivot_y_s1=FSTOrd50, _version_=BlockTreeOrds(blocksize=128), dense_pivot_x_s=Lucene50(blocksize=128), pivot_dt=FSTOrd50, multiDefault=PostingsFormat(name=Direct), dense_pivot_x_s1=PostingsFormat(name=Direct), pivot_x_s=Lucene50(blocksize=128), pivot_y_s=PostingsFormat(name=Direct), pivot_b1=PostingsFormat(name=Direct), pivot_ti1=Lucene50(blocksize=128), pivot_i=PostingsFormat(name=Direct), dense_pivot_ti=BlockTreeOrds(blocksize=128), dense_pivot_i1=FSTOrd50, intDefault=BlockTreeOrds(blocksize=128), dense_pivot_ti1=Lucene50(blocksize=128), pivot_tl=FSTOrd50, pivot_f=FSTOrd50, pivot_d1=BlockTreeOrds(blocksize=128), pivot_tdt=Lucene50(blocksize=128), pivot_x_s1=PostingsFormat(name=Direct), pivot_td1=PostingsFormat(name=Direct), timestamp=PostingsFormat(name=Direct), dense_pivot_y_s1=FSTOrd50, pivot_dt1=PostingsFormat(name=Direct), pivot_tf=Lucene50(blocksize=128), pivot_l1=BlockTreeOrds(blocksize=128), dense_pivot_i=PostingsFormat(name=Direct), pivot_d=Lucene50(blocksize=128), pivot_l=Lucene50(blocksize=128), pivot_td=FSTOrd50, pivot_i1=FSTOrd50}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=vi, timezone=Etc/GMT-11
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=88106264,total=255852544
   [junit4]   2> NOTE: All tests run in this JVM: [CoreAdminCreateDiscoverTest, TestReplicationHandlerBackup, TestReload, TestCryptoKeys, HdfsChaosMonkeySafeLeaderTest, UUIDUpdateProcessorFallbackTest, StressHdfsTest, ShardSplitTest, SolrInfoMBeanTest, DistributedFacetPivotLongTailTest, TestMaxScoreQueryParser, SpatialRPTFieldTypeTest, TestCursorMarkWithoutUniqueKey, TestManagedResourceStorage, OverseerTest, SystemInfoHandlerTest, TestOmitPositions, PluginInfoTest, SolrXmlInZkTest, DOMUtilTest, BinaryUpdateRequestHandlerTest, TestExpandComponent, BadComponentTest, SolrCoreCheckLockOnStartupTest, DeleteInactiveReplicaTest, CircularListTest, TestQueryUtils, InfoHandlerTest, RemoteQueryErrorTest, TestCloudManagedSchema, TestCloudInspectUtil, ActionThrottleTest, TestElisionMultitermQuery, SuggestComponentTest, TestInitParams, RequiredFieldsTest, DistributedFacetPivotWhiteBoxTest, PingRequestHandlerTest, DocValuesMissingTest, SpellingQueryConverterTest, SuggesterTSTTest, TestRandomFaceting, LeaderFailoverAfterPartitionTest, DirectSolrSpellCheckerTest, TestIBSimilarityFactory, TestJsonFacets, BasicZkTest, TestFieldSortValues, CoreAdminHandlerTest, SampleTest, JSONWriterTest, TestImplicitCoreProperties, TestConfig, CollectionsAPIDistributedZkTest, TestBlobHandler, TestMiniSolrCloudCluster, TriLevelCompositeIdRoutingTest, AssignTest, TestClusterStateMutator, LeaderInitiatedRecoveryOnCommitTest, RecoveryAfterSoftCommitTest, TestCloudPivotFacet]
   [junit4] Completed on J1 in 52.60s, 1 test, 1 failure <<< FAILURES!

[...truncated 685 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 1 failure, 61 ignored (21 assumptions)

Total time: 52 minutes 16 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 231424647 bytes
Compression is 0.0%
Took 43 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2889 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2889/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:26971/pq_dze/w/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:26971/pq_dze/w/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([F2CEA2424A880FB9:7A9A9D98E4746241]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:26971 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:28789 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:28789 failed to respond
	at __randomizedtesting.SeedInfo.seed([F2CEA2424A880FB9:7A9A9D98E4746241]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:28789 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:59378 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:59378 failed to respond
	at __randomizedtesting.SeedInfo.seed([F2CEA2424A880FB9:7A9A9D98E4746241]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:59378 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9445 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/init-core-data-001
   [junit4]   2> 226670 T1551 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 226676 T1551 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 226676 T1552 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 226776 T1551 oasc.ZkTestServer.run start zk server on port:28776
   [junit4]   2> 226777 T1551 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 226777 T1551 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 226781 T1559 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@73c8cb12 name:ZooKeeperConnection Watcher:127.0.0.1:28776 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 226781 T1551 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 226781 T1551 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 226782 T1551 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 226784 T1551 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 226784 T1551 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 226786 T1562 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@db0784c name:ZooKeeperConnection Watcher:127.0.0.1:28776/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 226786 T1551 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 226786 T1551 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 226786 T1551 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 226789 T1551 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 226790 T1551 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 226791 T1551 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 226793 T1551 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 226794 T1551 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 226796 T1551 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 226796 T1551 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 226798 T1551 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 226799 T1551 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 226800 T1551 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 226801 T1551 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 226802 T1551 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 226803 T1551 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 226804 T1551 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 226805 T1551 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 226806 T1551 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 226807 T1551 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 226808 T1551 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 226809 T1551 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 226810 T1551 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 226811 T1551 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 226812 T1551 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 226812 T1551 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 226814 T1551 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 226814 T1551 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 228002 T1551 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/cores/collection1
   [junit4]   2> 228006 T1551 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 228009 T1551 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:28785
   [junit4]   2> 228009 T1551 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, hostPort=28784, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/tempDir-001/control/data}
   [junit4]   2> 228010 T1551 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 228010 T1551 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/'
   [junit4]   2> 228039 T1551 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/solr.xml
   [junit4]   2> 228074 T1551 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/cores
   [junit4]   2> 228075 T1551 oasc.CoreContainer.<init> New CoreContainer 876967474
   [junit4]   2> 228075 T1551 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/]
   [junit4]   2> 228075 T1551 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/lib
   [junit4]   2> 228076 T1551 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/lib).
   [junit4]   2> 228084 T1551 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 228084 T1551 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 228085 T1551 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 228085 T1551 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 228085 T1551 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 228085 T1551 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 228085 T1551 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 228086 T1551 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 228086 T1551 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 228086 T1551 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 228086 T1551 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 228091 T1551 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 228092 T1551 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 228092 T1551 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 228092 T1551 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 228092 T1551 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:28776/solr
   [junit4]   2> 228093 T1551 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 228093 T1551 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 228093 T1551 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 228095 T1574 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3e183ec4 name:ZooKeeperConnection Watcher:127.0.0.1:28776 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 228096 T1551 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 228096 T1551 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 228098 T1551 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 228099 T1577 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1a8f6ee3 name:ZooKeeperConnection Watcher:127.0.0.1:28776/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 228100 T1551 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 228101 T1551 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 228104 T1551 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 228106 T1551 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 228108 T1551 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 228110 T1551 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 228113 T1551 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 228114 T1551 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 228116 T1551 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 228117 T1551 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:28784_
   [junit4]   2> 228118 T1551 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:28784_
   [junit4]   2> 228119 T1551 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 228121 T1551 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 228122 T1551 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 228124 T1551 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:28784_
   [junit4]   2> 228124 T1551 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 228126 T1551 oasc.Overseer.start Overseer (id=93595230515494915-127.0.0.1:28784_-n_0000000000) starting
   [junit4]   2> 228128 T1551 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 228135 T1551 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 228135 T1579 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 228135 T1551 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 228136 T1578 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 228139 T1551 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/cores
   [junit4]   2> 228141 T1551 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=control_collection, name=collection1, coreNodeName=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/cores/collection1/, config=solrconfig.xml, dataDir=data/, loadOnStartup=true, transient=false, shard=, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/cores/collection1}
   [junit4]   2> 228141 T1551 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/cores/collection1/
   [junit4]   2> 228142 T1551 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 228142 T1581 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 228142 T1581 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 228143 T1577 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 228143 T1581 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 228144 T1578 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28784",
   [junit4]   2> 	  "node_name":"127.0.0.1:28784_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1"} current state version: 0
   [junit4]   2> 228145 T1578 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28784",
   [junit4]   2> 	  "node_name":"127.0.0.1:28784_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 228145 T1578 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 228146 T1578 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 228147 T1577 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 229144 T1581 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 229145 T1581 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 229146 T1581 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 229146 T1581 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 229147 T1581 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 229147 T1581 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/cores/collection1/'
   [junit4]   2> 229164 T1581 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 229166 T1581 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 229178 T1581 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 229199 T1581 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 229247 T1581 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 229248 T1581 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 229259 T1581 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 229555 T1581 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 229566 T1581 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 229568 T1581 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 229580 T1581 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 229583 T1581 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 229586 T1581 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 229587 T1581 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 229587 T1581 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 229587 T1581 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 229588 T1581 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 229588 T1581 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 229588 T1581 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 229588 T1581 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 229588 T1581 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/cores/collection1/data/
   [junit4]   2> 229589 T1581 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@59e3c35d
   [junit4]   2> 229589 T1581 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/cores/collection1/data
   [junit4]   2> 229590 T1581 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/cores/collection1/data/index/
   [junit4]   2> 229590 T1581 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 229590 T1581 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/cores/collection1/data/index
   [junit4]   2> 229590 T1581 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=47, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 229591 T1581 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4fcf9353 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@44d15714),segFN=segments_1,generation=1}
   [junit4]   2> 229591 T1581 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 229596 T1581 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 229596 T1581 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 229596 T1581 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 229596 T1581 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 229596 T1581 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 229597 T1581 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 229597 T1581 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 229597 T1581 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 229597 T1581 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 229609 T1581 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 229611 T1581 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 229612 T1581 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 229614 T1581 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 229618 T1581 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/system,/admin/mbeans,standard,/admin/plugins,/replication,/admin/file,/update/json/docs,/admin/logging,/admin/properties,/update,/admin/ping,/admin/threads,/get,/config,/schema,/update/csv,/admin/luke,/update/json,/admin/segments
   [junit4]   2> 229620 T1581 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 229621 T1581 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 229621 T1581 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 229622 T1581 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 229622 T1581 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 229623 T1581 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=47, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.2593310496881438]
   [junit4]   2> 229623 T1581 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4fcf9353 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@44d15714),segFN=segments_1,generation=1}
   [junit4]   2> 229623 T1581 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 229623 T1581 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@367eb371[collection1] main
   [junit4]   2> 229624 T1581 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 229625 T1581 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 229625 T1581 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 229625 T1581 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 229625 T1581 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 229625 T1581 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 229626 T1581 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 229626 T1581 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 229626 T1581 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 229627 T1581 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 229627 T1582 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@367eb371[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 229628 T1581 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 229629 T1585 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:28784 collection:control_collection shard:shard1
   [junit4]   2> 229629 T1551 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 229629 T1551 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 229630 T1585 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 229634 T1585 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 229634 T1551 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 229635 T1551 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 229636 T1577 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 229636 T1585 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 229636 T1589 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6bfdaa4c name:ZooKeeperConnection Watcher:127.0.0.1:28776/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 229636 T1585 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 229637 T1578 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 229637 T1551 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> ASYNC  NEW_CORE C210 name=collection1 org.apache.solr.core.SolrCore@61dab312 url=http://127.0.0.1:28784/collection1 node=127.0.0.1:28784_ C210_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:28784, node_name=127.0.0.1:28784_, state=down, core=collection1}
   [junit4]   2> 229637 T1585 C:control_collection S:shard1 C210 P28784 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:28784/collection1/
   [junit4]   2> 229637 T1551 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 229638 T1585 C:control_collection S:shard1 C210 P28784 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 229638 T1551 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 229638 T1585 C:control_collection S:shard1 C210 P28784 oasc.SyncStrategy.syncToMe http://127.0.0.1:28784/collection1/ has no replicas
   [junit4]   2> 229639 T1585 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:28784/collection1/ shard1
   [junit4]   2> 229639 T1585 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 229641 T1551 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 229643 T1577 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 229643 T1578 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28784",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 1
   [junit4]   2> 229748 T1577 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 229748 T1589 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 229795 T1585 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:28784/collection1/ and leader is http://127.0.0.1:28784/collection1/
   [junit4]   2> 229796 T1585 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:28784
   [junit4]   2> 229796 T1585 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 229796 T1585 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 229796 T1585 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 229798 T1577 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 229799 T1578 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28784",
   [junit4]   2> 	  "node_name":"127.0.0.1:28784_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core":"collection1"} current state version: 2
   [junit4]   2> 229800 T1578 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28784",
   [junit4]   2> 	  "node_name":"127.0.0.1:28784_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 229904 T1589 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 229904 T1577 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 230855 T1551 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/cores/collection1
   [junit4]   2> 230857 T1551 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001
   [junit4]   2> 230858 T1551 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 230860 T1551 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:28790
   [junit4]   2> 230860 T1551 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/tempDir-001/jetty1, hostPort=28789, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/cores, solrconfig=solrconfig.xml}
   [junit4]   2> 230861 T1551 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 230861 T1551 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/'
   [junit4]   2> 230902 T1551 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/solr.xml
   [junit4]   2> 230953 T1551 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/cores
   [junit4]   2> 230954 T1551 oasc.CoreContainer.<init> New CoreContainer 420523198
   [junit4]   2> 230954 T1551 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/]
   [junit4]   2> 230954 T1551 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/lib
   [junit4]   2> 230955 T1551 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/lib).
   [junit4]   2> 230964 T1551 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 230965 T1551 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 230965 T1551 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 230965 T1551 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 230965 T1551 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 230965 T1551 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 230966 T1551 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 230966 T1551 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 230966 T1551 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 230966 T1551 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 230967 T1551 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 230973 T1551 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 230974 T1551 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 230974 T1551 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 230975 T1551 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 230975 T1551 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:28776/solr
   [junit4]   2> 230975 T1551 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 230975 T1551 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 230976 T1551 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 230978 T1601 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4d8fdec7 name:ZooKeeperConnection Watcher:127.0.0.1:28776 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 230978 T1551 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 230979 T1551 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 230980 T1551 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 230982 T1604 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@494d0e42 name:ZooKeeperConnection Watcher:127.0.0.1:28776/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 230982 T1551 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 230987 T1551 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 231991 T1551 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:28789_
   [junit4]   2> 231992 T1551 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:28789_
   [junit4]   2> 231994 T1551 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 231996 T1551 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/cores
   [junit4]   2> 231998 T1551 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=collection1, transient=false, dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/cores/collection1/, schema=schema.xml, shard=, coreNodeName=, name=collection1, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/cores/collection1, loadOnStartup=true}
   [junit4]   2> 231998 T1551 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/cores/collection1/
   [junit4]   2> 231998 T1551 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 231999 T1605 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 231999 T1605 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 232000 T1577 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 232000 T1605 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 232001 T1578 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28789",
   [junit4]   2> 	  "node_name":"127.0.0.1:28789_",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1"} current state version: 3
   [junit4]   2> 232001 T1578 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28789",
   [junit4]   2> 	  "node_name":"127.0.0.1:28789_",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 232002 T1578 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 232002 T1578 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 232003 T1577 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 232003 T1604 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 232003 T1589 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 233000 T1605 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 233001 T1605 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 233001 T1605 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 233002 T1605 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 233002 T1605 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 233002 T1605 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/cores/collection1/'
   [junit4]   2> 233014 T1605 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 233017 T1605 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 233024 T1605 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 233039 T1605 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 233072 T1605 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 233073 T1605 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 233080 T1605 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 233346 T1605 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 233357 T1605 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 233359 T1605 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 233371 T1605 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 233374 T1605 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 233377 T1605 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 233378 T1605 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 233378 T1605 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 233378 T1605 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 233379 T1605 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 233379 T1605 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 233379 T1605 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 233380 T1605 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 233380 T1605 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 233380 T1605 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@59e3c35d
   [junit4]   2> 233381 T1605 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/cores/collection1/data
   [junit4]   2> 233381 T1605 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 233381 T1605 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 233381 T1605 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 233382 T1605 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=47, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 233382 T1605 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@51965c17 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@667813d9),segFN=segments_1,generation=1}
   [junit4]   2> 233382 T1605 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 233388 T1605 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 233388 T1605 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 233388 T1605 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 233389 T1605 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 233389 T1605 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 233389 T1605 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 233389 T1605 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 233390 T1605 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 233390 T1605 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 233402 T1605 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 233403 T1605 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 233405 T1605 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 233407 T1605 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 233412 T1605 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/system,/admin/mbeans,standard,/admin/plugins,/replication,/admin/file,/update/json/docs,/admin/logging,/admin/properties,/update,/admin/ping,/admin/threads,/get,/config,/schema,/update/csv,/admin/luke,/update/json,/admin/segments
   [junit4]   2> 233413 T1605 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 233414 T1605 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 233415 T1605 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 233416 T1605 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 233416 T1605 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 233417 T1605 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=47, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.2593310496881438]
   [junit4]   2> 233417 T1605 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@51965c17 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@667813d9),segFN=segments_1,generation=1}
   [junit4]   2> 233417 T1605 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 233417 T1605 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@2e46cd5b[collection1] main
   [junit4]   2> 233418 T1605 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 233419 T1605 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 233419 T1605 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 233419 T1605 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 233420 T1605 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 233420 T1605 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 233420 T1605 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 233421 T1605 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 233421 T1605 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 233421 T1605 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 233422 T1606 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2e46cd5b[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 233423 T1605 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 233423 T1609 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:28789 collection:collection1 shard:shard1
   [junit4]   2> 233424 T1551 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 233424 T1551 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 233424 T1609 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 233429 T1609 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 233431 T1577 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 233431 T1609 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 233431 T1609 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C211 name=collection1 org.apache.solr.core.SolrCore@2c12d853 url=http://127.0.0.1:28789/collection1 node=127.0.0.1:28789_ C211_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:28789, node_name=127.0.0.1:28789_, state=down, core=collection1}
   [junit4]   2> 233432 T1609 C:collection1 S:shard1 C211 P28789 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:28789/collection1/
   [junit4]   2> 233432 T1578 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 4
   [junit4]   2> 233432 T1609 C:collection1 S:shard1 C211 P28789 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 233432 T1609 C:collection1 S:shard1 C211 P28789 oasc.SyncStrategy.syncToMe http://127.0.0.1:28789/collection1/ has no replicas
   [junit4]   2> 233433 T1609 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:28789/collection1/ shard1
   [junit4]   2> 233433 T1609 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 233437 T1577 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 233438 T1578 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28789",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 4
   [junit4]   2> 233541 T1577 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 233541 T1604 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 233541 T1589 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 233590 T1609 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:28789/collection1/ and leader is http://127.0.0.1:28789/collection1/
   [junit4]   2> 233591 T1609 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:28789
   [junit4]   2> 233591 T1609 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 233591 T1609 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 233591 T1609 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 233592 T1577 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 233593 T1578 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28789",
   [junit4]   2> 	  "node_name":"127.0.0.1:28789_",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core":"collection1"} current state version: 5
   [junit4]   2> 233594 T1578 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28789",
   [junit4]   2> 	  "node_name":"127.0.0.1:28789_",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 233697 T1577 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 233697 T1589 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 233697 T1604 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 234946 T1551 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-2-001/cores/collection1
   [junit4]   2> 234949 T1551 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-2-001
   [junit4]   2> 234949 T1551 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 234951 T1551 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:28798
   [junit4]   2> 234952 T1551 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-2-001/cores, hostPort=28797, solrconfig=solrconfig.xml, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/tempDir-001/jetty2, hostContext=/}
   

[...truncated too long message...]

solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:142)
   [junit4]   2> 		... 11 more
   [junit4]   2> 	
   [junit4]   2> 1213402 T2415 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1213404 T2198 C:control_collection S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/pq_dze/w,null}
   [junit4]   2> 1213426 T2198 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:26935 26935
   [junit4]   2> 1213524 T2199 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:26935 26935
   [junit4]   2> 1213525 T2199 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/queue
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1213526 T2198 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 11 connections to: http://127.0.0.1:26947/pq_dze/w
   [junit4]   2> 1213526 T2198 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:26941/pq_dze/w
   [junit4]   2> 1213526 T2198 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:26953/pq_dze/w
   [junit4]   2> 1213527 T2198 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:26982/pq_dze/w
   [junit4]   2> 1213527 T2198 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:26971/pq_dze/w
   [junit4]   2> 1213528 T2198 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:26947/pq_dze/w
   [junit4]   2> 1213528 T2198 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:26941/pq_dze/w
   [junit4]   2> 1213528 T2198 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:26953/pq_dze/w
   [junit4]   2> 1213528 T2198 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:26982/pq_dze/w
   [junit4]   2> 1213529 T2198 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:26971/pq_dze/w
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=F2CEA2424A880FB9 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=hu_HU -Dtests.timezone=America/Barbados -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   76.7s J2 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:26971/pq_dze/w/c8n_1x3_commits_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([F2CEA2424A880FB9:7A9A9D98E4746241]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:26971 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 48 more
   [junit4]   2> 1213549 T2198 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F2CEA2424A880FB9-001
   [junit4]   2> 76699 T2197 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=hu_HU, timezone=America/Barbados
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=188234224,total=325582848
   [junit4]   2> NOTE: All tests run in this JVM: [NoCacheHeaderTest, TestManagedSchemaFieldResource, TestRecoveryHdfs, TestDistributedSearch, DocumentAnalysisRequestHandlerTest, TestStandardQParsers, TestReload, DocValuesMissingTest, RegexBoostProcessorTest, TestFunctionQuery, TestStressVersions, TestXIncludeConfig, TestSolrXml, TestInitParams, ChaosMonkeySafeLeaderTest, TestBlendedInfixSuggestions, TestPerFieldSimilarity, RecoveryZkTest, TestBadConfig, TestRebalanceLeaders, SyncSliceTest, AssignTest, SaslZkACLProviderTest, TestHighlightDedupGrouping, TestLMJelinekMercerSimilarityFactory, DistribJoinFromCollectionTest, DistributedMLTComponentTest, MigrateRouteKeyTest, DocumentBuilderTest, TestSchemaVersionResource, OverseerStatusTest, OpenExchangeRatesOrgProviderTest, TestExtendedDismaxParser, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J2 in 77.24s, 1 test, 1 error <<< FAILURES!

[...truncated 867 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 54 minutes 41 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 222602195 bytes
Compression is 0.0%
Took 42 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2888 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2888/

4 tests failed.
REGRESSION:  org.apache.solr.cloud.RecoveryZkTest.test

Error Message:
shard1 is not consistent.  Got 781 from https://127.0.0.1:35615/x_qkw/collection1lastClient and got 165 from https://127.0.0.1:35630/x_qkw/collection1

Stack Trace:
java.lang.AssertionError: shard1 is not consistent.  Got 781 from https://127.0.0.1:35615/x_qkw/collection1lastClient and got 165 from https://127.0.0.1:35630/x_qkw/collection1
	at __randomizedtesting.SeedInfo.seed([88D46D6800DE846B:8052B2AE22E993]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:123)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:62554/kz/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:62554/kz/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([88D46D6800DE846B:8052B2AE22E993]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:62554 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:18591/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:18591/collection1
	at __randomizedtesting.SeedInfo.seed([88D46D6800DE846B:8052B2AE22E993]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:18591/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:18591 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:44740/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:44740/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([88D46D6800DE846B:8052B2AE22E993]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:44740/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:44740 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9325 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/init-core-data-001
   [junit4]   2> 35157 T48 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /kz/
   [junit4]   2> 35233 T48 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 35245 T49 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 35647 T48 oasc.ZkTestServer.run start zk server on port:20767
   [junit4]   2> 35833 T48 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 36134 T48 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 36342 T56 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@a590747 name:ZooKeeperConnection Watcher:127.0.0.1:20767 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 36343 T48 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 36351 T48 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 36365 T48 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 36505 T48 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 36507 T48 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 36515 T59 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@12e82c49 name:ZooKeeperConnection Watcher:127.0.0.1:20767/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 36516 T48 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 36517 T48 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 36534 T48 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 36540 T48 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 36545 T48 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 36549 T48 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 36554 T48 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 36587 T48 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 36597 T48 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 36598 T48 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 36603 T48 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 36604 T48 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 36609 T48 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 36610 T48 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 36614 T48 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 36615 T48 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 36619 T48 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 36620 T48 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 36624 T48 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 36625 T48 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 36629 T48 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 36630 T48 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 36633 T48 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 36634 T48 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 36637 T48 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 36638 T48 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 36642 T48 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 36642 T48 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 38352 T48 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/cores/collection1
   [junit4]   2> 38358 T48 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 38361 T48 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:39301
   [junit4]   2> 38361 T48 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/kz, hostPort=11539, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/cores}
   [junit4]   2> 38362 T48 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 38363 T48 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/'
   [junit4]   2> 38419 T48 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/solr.xml
   [junit4]   2> 38504 T48 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/cores
   [junit4]   2> 38505 T48 oasc.CoreContainer.<init> New CoreContainer 1362813671
   [junit4]   2> 38505 T48 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/]
   [junit4]   2> 38505 T48 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/lib
   [junit4]   2> 38507 T48 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/lib).
   [junit4]   2> 38521 T48 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 38521 T48 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 38522 T48 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 38522 T48 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 38523 T48 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 38523 T48 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 38523 T48 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 38524 T48 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 38524 T48 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 38525 T48 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 38525 T48 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 38536 T48 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 38537 T48 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 38538 T48 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 38538 T48 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 38538 T48 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:20767/solr
   [junit4]   2> 38618 T48 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 38619 T48 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 38620 T48 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 38622 T71 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@400f1a30 name:ZooKeeperConnection Watcher:127.0.0.1:20767 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 38623 T48 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 38624 T48 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 38662 T48 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 38664 T74 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@9a83d99 name:ZooKeeperConnection Watcher:127.0.0.1:20767/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 38664 T48 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 38695 T48 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 38719 T48 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 38730 T48 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 38735 T48 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 38739 T48 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 38774 T48 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 38778 T48 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 38781 T48 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 38783 T48 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:11539_kz
   [junit4]   2> 38795 T48 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:11539_kz
   [junit4]   2> 38800 T48 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 38804 T48 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 38808 T48 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 38836 T48 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:11539_kz
   [junit4]   2> 38837 T48 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 38839 T48 oasc.Overseer.start Overseer (id=93594278762905603-127.0.0.1:11539_kz-n_0000000000) starting
   [junit4]   2> 38869 T48 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 38908 T48 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 39452 T76 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 39452 T48 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 39493 T75 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 39506 T48 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/cores
   [junit4]   2> 39533 T48 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, loadOnStartup=true, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/cores/collection1/, dataDir=data/, collection=control_collection, schema=schema.xml, name=collection1, coreNodeName=, config=solrconfig.xml, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/cores/collection1}
   [junit4]   2> 39533 T48 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/cores/collection1/
   [junit4]   2> 39534 T48 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 39536 T78 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 39537 T78 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 39567 T74 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 39567 T78 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 39579 T75 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11539/kz",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:11539_kz",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null} current state version: 0
   [junit4]   2> 39673 T75 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11539/kz",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:11539_kz",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 39684 T75 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 39777 T75 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 39807 T74 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 40570 T78 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 40570 T78 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 40572 T78 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 40572 T78 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 40574 T78 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 40575 T78 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/cores/collection1/'
   [junit4]   2> 40645 T78 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 40673 T78 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 40730 T78 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 40787 T78 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 40902 T78 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 40905 T78 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 41064 T78 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 42901 T78 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 43008 T78 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 43018 T78 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 43383 T78 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 43424 T78 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 43431 T78 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 43441 T78 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 43441 T78 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 43442 T78 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 43444 T78 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 43444 T78 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 43464 T78 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 43465 T78 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 43466 T78 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/cores/collection1/data/
   [junit4]   2> 43481 T78 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5d098870
   [junit4]   2> 43652 T78 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/cores/collection1/data
   [junit4]   2> 43653 T78 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/cores/collection1/data/index/
   [junit4]   2> 43653 T78 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 43654 T78 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/cores/collection1/data/index
   [junit4]   2> 43655 T78 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=25, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=38.2470703125, floorSegmentMB=1.8984375, forceMergeDeletesPctAllowed=28.361484787913188, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 43656 T78 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@73dd66b6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4c827227),segFN=segments_1,generation=1}
   [junit4]   2> 43656 T78 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 43670 T78 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 43851 T78 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 43869 T78 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 43869 T78 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 43870 T78 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 43870 T78 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 44097 T78 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 44098 T78 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 44098 T78 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 44125 T78 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 44138 T78 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 44141 T78 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 44145 T78 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 44202 T78 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update,/admin/threads,/admin/luke,/admin/logging,/admin/ping,/replication,/update/csv,/update/json,/admin/segments,/admin/mbeans,/update/json/docs,/admin/properties,/admin/file,/config,/admin/system,/schema,/admin/plugins,standard,/get
   [junit4]   2> 44211 T78 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 44234 T78 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 44234 T78 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 44256 T78 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 44257 T78 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 44258 T78 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=45, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=65.017578125, floorSegmentMB=1.97265625, forceMergeDeletesPctAllowed=18.72732245389139, segmentsPerTier=46.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8059727721030615
   [junit4]   2> 44260 T78 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@73dd66b6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4c827227),segFN=segments_1,generation=1}
   [junit4]   2> 44260 T78 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 44261 T78 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@68543a0[collection1] main
   [junit4]   2> 44261 T78 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 44263 T78 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 44264 T78 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 44282 T78 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 44283 T78 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 44284 T78 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 44285 T78 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 44286 T78 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 44287 T78 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 44336 T78 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 44341 T79 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@68543a0[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 44348 T78 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 44349 T82 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:11539/kz collection:control_collection shard:shard1
   [junit4]   2> 44350 T48 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 44351 T48 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 44421 T82 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 44431 T82 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 44435 T74 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 44436 T82 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 44436 T82 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 44437 T75 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C4 name=collection1 org.apache.solr.core.SolrCore@291d335e url=http://127.0.0.1:11539/kz/collection1 node=127.0.0.1:11539_kz C4_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:11539/kz, core=collection1, state=down, node_name=127.0.0.1:11539_kz}
   [junit4]   2> 44437 T82 C:control_collection S:shard1 C4 P11539 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:11539/kz/collection1/
   [junit4]   2> 44439 T82 C:control_collection S:shard1 C4 P11539 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 44439 T82 C:control_collection S:shard1 C4 P11539 oasc.SyncStrategy.syncToMe http://127.0.0.1:11539/kz/collection1/ has no replicas
   [junit4]   2> 44440 T82 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:11539/kz/collection1/ shard1
   [junit4]   2> 44451 T74 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 44456 T48 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 44457 T48 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 44459 T86 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4aa67956 name:ZooKeeperConnection Watcher:127.0.0.1:20767/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 44460 T48 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 44461 T48 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 44462 T48 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 44462 T82 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 44471 T74 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 44483 T48 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 44484 T75 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11539/kz",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 44591 T74 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 44591 T86 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 44636 T82 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:11539/kz/collection1/ and leader is http://127.0.0.1:11539/kz/collection1/
   [junit4]   2> 44645 T82 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:11539/kz
   [junit4]   2> 44645 T82 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 44646 T82 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 44646 T82 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 44648 T74 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 44651 T75 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11539/kz",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:11539_kz",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1"} current state version: 3
   [junit4]   2> 44652 T75 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11539/kz",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:11539_kz",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 44759 T86 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 44759 T74 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 46370 T48 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/cores/collection1
   [junit4]   2> 46373 T48 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001
   [junit4]   2> 46374 T48 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 46377 T48 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:19938
   [junit4]   2> 46377 T48 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=40835, hostContext=/kz, solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/tempDir-001/jetty1}
   [junit4]   2> 46378 T48 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 46379 T48 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/'
   [junit4]   2> 46424 T48 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/solr.xml
   [junit4]   2> 46486 T48 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/cores
   [junit4]   2> 46487 T48 oasc.CoreContainer.<init> New CoreContainer 1626291339
   [junit4]   2> 46487 T48 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/]
   [junit4]   2> 46488 T48 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/lib
   [junit4]   2> 46489 T48 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/lib).
   [junit4]   2> 46503 T48 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 46503 T48 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 46504 T48 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 46504 T48 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 46505 T48 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 46505 T48 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 46506 T48 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 46506 T48 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 46507 T48 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 46507 T48 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 46508 T48 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 46515 T48 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 46516 T48 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 46517 T48 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 46517 T48 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 46517 T48 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:20767/solr
   [junit4]   2> 46518 T48 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 46518 T48 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 46519 T48 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 46522 T98 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@203ad0b0 name:ZooKeeperConnection Watcher:127.0.0.1:20767 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 46523 T48 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 46524 T48 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 46527 T48 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 46529 T101 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@294d765e name:ZooKeeperConnection Watcher:127.0.0.1:20767/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 46529 T48 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 46539 T48 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 47548 T48 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40835_kz
   [junit4]   2> 47550 T48 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:40835_kz
   [junit4]   2> 47555 T48 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 47569 T48 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/cores
   [junit4]   2> 47572 T48 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {loadOnStartup=true, coreNodeName=, shard=, schema=schema.xml, transient=false, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/cores/collection1/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/cores/collection1, dataDir=data/, name=collection1, collection=collection1}
   [junit4]   2> 47573 T48 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/cores/collection1/
   [junit4]   2> 47574 T48 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 47577 T102 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 47577 T102 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 47579 T74 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 47579 T102 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 47581 T75 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:40835/kz",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:40835_kz",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null} current state version: 4
   [junit4]   2> 47582 T75 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:40835/kz",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:40835_kz",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 47583 T75 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 47584 T75 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 47587 T86 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 47587 T74 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 47587 T101 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 48580 T102 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 48582 T102 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 48583 T102 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 48584 T102 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 48586 T102 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 48586 T102 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/cores/collection1/'
   [junit4]   2> 48613 T102 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 48618 T102 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 48658 T102 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 48697 T102 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 48830 T102 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 48833 T102 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 48921 T102 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 49384 T102 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 49433 T102 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 49437 T102 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 49474 T102 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 49480 T102 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 49486 T102 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 49488 T102 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 49489 T102 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 49489 T102 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 49491 T102 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 49491 T102 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 49492 T102 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 49492 T102 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 49492 T102 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 49493 T102 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5d098870
   [junit4]   2> 49494 T102 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/cores/collection1/data
   [junit4]   2> 49495 T102 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 49495 T102 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 49496 T102 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 49496 T102 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=25, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=38.2470703125, floorSegmentMB=1.8984375, forceMergeDeletesPctAllowed=28.361484787913188, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 49497 T102 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1cf0850b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@72aea404),segFN=segments_1,generation=1}
   [junit4]   2> 49498 T102 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 49510 T102 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 49511 T102 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 49512 T102 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 49512 T102 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 49513 T102 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 49513 T102 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 49514 T102 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 49514 T102 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 49514 T102 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 49542 T102 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 49547 T102 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 49550 T102 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 49554 T102 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 49576 T102 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update,/admin/threads,/admin/luke,/admin/logging,/admin/ping,/replication,/update/csv,/update/json,/admin/segments,/admin/mbeans,/update/json/docs,/admin/properties,/admin/file,/config,/admin/system,/schema,/admin/plugins,standard,/get
   [junit4]   2> 49579 T102 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 49581 T102 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 49582 T102 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 49583 T102 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 49584 T102 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 49586 T102 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=45, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=65.017578125, floorSegmentMB=1.97265625, forceMergeDeletesPctAllowed=18.72732245389139, segmentsPerTier=46.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8059727721030615
   [junit4]   2> 49587 T102 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1cf0850b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@72aea404),segFN=segments_1,generation=1}
   [junit4]   2> 49587 T102 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 49588 T102 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@30eb5ea0[collection1] main
   [junit4]   2> 49588 T102 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 49590 T102 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 49590 T102 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 49591 T102 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 49591 T102 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 49592 T102 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 49593 T102 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 49593 T102 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 49594 T102 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 49594 T102 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 49598 T103 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@30eb5ea0[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 49599 T102 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 49601 T106 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:40835/kz collection:collection1 shard:shard1
   [junit4]   2> 49601 T48 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 49602 T48 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 49602 T106 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 49611 T106 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 49614 T74 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 49614 T106 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 49615 T106 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C5 name=collection1 org.apache.solr.core.SolrCore@2f4f13b8 url=http://127.0.0.1:40835/kz/collection1 node=127.0.0.1:40835_kz C5_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:40835/kz, core=collection1, state=down, node_name=127.0.0.1:40835_kz}
   [junit4]   2> 49615 T106 C:collection1 S:shard1 C5 P40835 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:40835/kz/collection1/
   [junit4]   2> 49616 T75 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 49616 T106 C:collection1 S:shard1 C5 P40835 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 49617 T106 C:collection1 S:shard1 C5 P40835 oasc.SyncStrategy.syncToMe http://127.0.0.1:40835/kz/collection1/ has no replicas
   [junit4]   2> 49618 T106 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:40835/kz/collection1/ shard1
   [junit4]   2> 49619 T106 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 49620 T101 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 49620 T74 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent stat

[...truncated too long message...]

0.1:44740/
   [junit4]   2> 1484626 T3118 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:44702/
   [junit4]   2> 1484626 T3118 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:44692/
   [junit4]   2> 1484627 T3118 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:44748/
   [junit4]   2> 1484627 T3118 C:control_collection S:shard1 c:collection1 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1484627 T3118 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:44740/
   [junit4]   2> 1484628 T3118 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:44702/
   [junit4]   2> 1484628 T3118 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:44692/
   [junit4]   2> 1484628 T3118 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:44748/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=88D46D6800DE846B -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ar_LB -Dtests.timezone=Canada/Newfoundland -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   76.7s J1 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:44740/repfacttest_c8n_1x3_shard1_replica3
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([88D46D6800DE846B:8052B2AE22E993]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:44740/repfacttest_c8n_1x3_shard1_replica3
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:44740 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 53 more
   [junit4]   2> 1484655 T3118 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 88D46D6800DE846B-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {intDefault=PostingsFormat(name=Direct), text=PostingsFormat(name=LuceneVarGapDocFreqInterval), multiDefault=Lucene50(blocksize=128), timestamp=Lucene50(blocksize=128), id=PostingsFormat(name=Memory doPackFST= true), a_t=Lucene50(blocksize=128), _version_=PostingsFormat(name=Direct), range_facet_l=Lucene50(blocksize=128)}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=RandomSimilarityProvider(queryNorm=false,coord=crazy): {}, locale=ar_LB, timezone=Canada/Newfoundland
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=66636552,total=305659904
   [junit4]   2> NOTE: All tests run in this JVM: [DistanceUnitsTest, TestSchemaVersionResource, DeleteReplicaTest, CoreAdminCreateDiscoverTest, TermsComponentTest, TestRandomFaceting, BadComponentTest, PolyFieldTest, IndexSchemaTest, TestCoreContainer, TestBulkSchemaConcurrent, TestWriterPerf, NoCacheHeaderTest, QueryEqualityTest, RecoveryZkTest, TestRecoveryHdfs, HdfsSyncSliceTest, DistribJoinFromCollectionTest, TestPivotHelperCode, DistributedSpellCheckComponentTest, TestBadConfig, SuggestComponentTest, TestManagedResourceStorage, RecoveryAfterSoftCommitTest, OpenExchangeRatesOrgProviderTest, SSLMigrationTest, DistanceFunctionTest, DistributedQueryElevationComponentTest, TestConfig, TestExactStatsCache, ScriptEngineTest, TestJmxMonitoredMap, TestConfigSets, SpellCheckCollatorTest, ResponseHeaderTest, HighlighterConfigTest, ResponseLogComponentTest, BasicFunctionalityTest, TestDFRSimilarityFactory, AddBlockUpdateTest, HardAutoCommitTest, ShardSplitTest, AsyncMigrateRouteKeyTest, TestSolrConfigHandlerConcurrent, AnalyticsQueryTest, EchoParamsTest, DirectUpdateHandlerOptimizeTest, SaslZkACLProviderTest, DistributedFacetPivotWhiteBoxTest, TestFieldResource, BJQParserTest, LeaderFailoverAfterPartitionTest, CursorPagingTest, MoreLikeThisHandlerTest, TestConfigOverlay, TestReloadAndDeleteDocs, TestClusterStateMutator, SpellCheckComponentTest, TestJmxIntegration, ReplicationFactorTest]
   [junit4] Completed on J1 in 76.76s, 1 test, 1 error <<< FAILURES!

[...truncated 721 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 1 failure, 61 ignored (21 assumptions)

Total time: 54 minutes 51 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 214173162 bytes
Compression is 0.0%
Took 43 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2887 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2887/

4 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:64721/xiwg/br/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:64721/xiwg/br/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([E317CCBEAFC08265:6B43F364013CEF9D]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:64721 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:57112 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:57112 failed to respond
	at __randomizedtesting.SeedInfo.seed([E317CCBEAFC08265:6B43F364013CEF9D]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:57112 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:35138 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:35138 failed to respond
	at __randomizedtesting.SeedInfo.seed([E317CCBEAFC08265:6B43F364013CEF9D]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:35138 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.handler.component.DistributedMLTComponentTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:59381//collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:59381//collection1
	at __randomizedtesting.SeedInfo.seed([E317CCBEAFC08265:6B43F364013CEF9D]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:566)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:558)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:606)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:588)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:567)
	at org.apache.solr.handler.component.DistributedMLTComponentTest.test(DistributedMLTComponentTest.java:126)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 51 more




Build Log:
[...truncated 9549 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/init-core-data-001
   [junit4]   2> 214318 T385 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /_/h
   [junit4]   2> 214326 T385 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 214327 T386 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 214427 T385 oasc.ZkTestServer.run start zk server on port:25045
   [junit4]   2> 214428 T385 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 214429 T385 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 214434 T393 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@719be136 name:ZooKeeperConnection Watcher:127.0.0.1:25045 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 214434 T385 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 214435 T385 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 214435 T385 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 214438 T385 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 214439 T385 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 214441 T396 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7b240a6a name:ZooKeeperConnection Watcher:127.0.0.1:25045/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 214441 T385 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 214441 T385 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 214442 T385 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 214444 T385 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 214446 T385 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 214447 T385 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 214449 T385 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 214449 T385 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 214452 T385 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 214452 T385 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 214454 T385 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 214455 T385 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 214456 T385 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 214457 T385 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 214458 T385 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 214459 T385 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 214460 T385 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 214460 T385 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 214462 T385 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 214462 T385 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 214464 T385 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 214464 T385 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 214466 T385 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 214466 T385 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 214468 T385 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 214468 T385 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 214469 T385 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 214470 T385 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 217283 T385 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/cores/collection1
   [junit4]   2> 217287 T385 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 217291 T385 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:27258
   [junit4]   2> 217291 T385 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/cores, hostContext=/_/h, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/tempDir-001/control/data, hostPort=31580}
   [junit4]   2> 217292 T385 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 217292 T385 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/'
   [junit4]   2> 217352 T385 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/solr.xml
   [junit4]   2> 217475 T385 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/cores
   [junit4]   2> 217475 T385 oasc.CoreContainer.<init> New CoreContainer 1201381070
   [junit4]   2> 217476 T385 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/]
   [junit4]   2> 217476 T385 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/lib
   [junit4]   2> 217478 T385 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/lib).
   [junit4]   2> 217493 T385 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 217493 T385 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 217494 T385 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 217494 T385 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 217494 T385 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 217495 T385 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 217495 T385 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 217495 T385 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 217496 T385 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 217496 T385 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 217496 T385 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 217509 T385 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 217510 T385 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 217511 T385 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 217511 T385 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 217512 T385 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:25045/solr
   [junit4]   2> 217512 T385 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 217513 T385 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 217514 T385 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 217516 T408 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@469f249f name:ZooKeeperConnection Watcher:127.0.0.1:25045 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 217517 T385 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 217518 T385 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 217521 T385 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 217523 T411 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@25ecfac5 name:ZooKeeperConnection Watcher:127.0.0.1:25045/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 217524 T385 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 217526 T385 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 217529 T385 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 217531 T385 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 217534 T385 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 217536 T385 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 217539 T385 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 217540 T385 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 217542 T385 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 217543 T385 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:31580__%2Fh
   [junit4]   2> 217544 T385 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:31580__%2Fh
   [junit4]   2> 217546 T385 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 217547 T385 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 217548 T385 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 217550 T385 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:31580__%2Fh
   [junit4]   2> 217550 T385 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 217552 T385 oasc.Overseer.start Overseer (id=93593343533056003-127.0.0.1:31580__%2Fh-n_0000000000) starting
   [junit4]   2> 217554 T385 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 217560 T385 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 217561 T413 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 217561 T385 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 217561 T412 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 217565 T385 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/cores
   [junit4]   2> 217569 T385 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/cores/collection1, collection=control_collection, schema=schema.xml, name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/cores/collection1/, dataDir=data/, coreNodeName=, loadOnStartup=true, transient=false}
   [junit4]   2> 217570 T385 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/cores/collection1/
   [junit4]   2> 217570 T385 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 217571 T415 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 217572 T415 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 217573 T415 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 217573 T411 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 217574 T412 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:31580/_/h",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:31580__%2Fh"} current state version: 0
   [junit4]   2> 217575 T412 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:31580/_/h",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:31580__%2Fh"}
   [junit4]   2> 217575 T412 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 217576 T412 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 217577 T411 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 218573 T415 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 218574 T415 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 218574 T415 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 218575 T415 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 218575 T415 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 218576 T415 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/cores/collection1/'
   [junit4]   2> 218630 T415 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 218633 T415 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 218657 T415 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 218709 T415 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 218791 T415 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 218793 T415 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 218825 T415 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 219485 T415 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 219515 T415 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 219518 T415 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 219534 T415 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 219540 T415 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 219545 T415 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 219546 T415 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 219547 T415 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 219547 T415 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 219548 T415 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 219548 T415 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 219549 T415 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 219549 T415 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 219549 T415 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/cores/collection1/data/
   [junit4]   2> 219550 T415 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@62c5dc6a
   [junit4]   2> 219551 T415 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/cores/collection1/data
   [junit4]   2> 219551 T415 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/cores/collection1/data/index/
   [junit4]   2> 219551 T415 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 219552 T415 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/cores/collection1/data/index
   [junit4]   2> 219552 T415 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=24, maxMergeAtOnceExplicit=29, maxMergedSegmentMB=3.1484375, floorSegmentMB=1.59375, forceMergeDeletesPctAllowed=20.062224976109842, segmentsPerTier=43.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 219553 T415 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@57f15d5d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2728b1c6),segFN=segments_1,generation=1}
   [junit4]   2> 219554 T415 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 219562 T415 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 219562 T415 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 219563 T415 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 219563 T415 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 219563 T415 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 219564 T415 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 219564 T415 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 219565 T415 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 219565 T415 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 219608 T415 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 219611 T415 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 219614 T415 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 219616 T415 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 219624 T415 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /config,/admin/plugins,/get,/update/json/docs,/admin/ping,/update/csv,/schema,/admin/segments,/admin/system,/admin/file,/admin/mbeans,/admin/luke,standard,/admin/logging,/admin/threads,/update,/replication,/admin/properties,/update/json
   [junit4]   2> 219646 T415 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 219649 T415 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 219649 T415 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 219651 T415 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 219652 T415 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 219653 T415 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=37, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 219653 T415 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@57f15d5d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2728b1c6),segFN=segments_1,generation=1}
   [junit4]   2> 219654 T415 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 219654 T415 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@6648ff7e[collection1] main
   [junit4]   2> 219655 T415 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 219656 T415 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 219656 T415 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 219656 T415 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 219657 T415 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 219657 T415 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 219658 T415 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 219658 T415 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 219658 T415 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 219659 T415 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 219660 T416 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6648ff7e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 219661 T415 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 219661 T419 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:31580/_/h collection:control_collection shard:shard1
   [junit4]   2> 219662 T385 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 219662 T385 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 219663 T419 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 219668 T419 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 219669 T385 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 219669 T411 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 219670 T419 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 219670 T419 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 219670 T385 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> ASYNC  NEW_CORE C63 name=collection1 org.apache.solr.core.SolrCore@63089797 url=http://127.0.0.1:31580/_/h/collection1 node=127.0.0.1:31580__%2Fh C63_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:31580/_/h, state=down, core=collection1, node_name=127.0.0.1:31580__%2Fh}
   [junit4]   2> 219670 T419 C:control_collection S:shard1 C63 P31580 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:31580/_/h/collection1/
   [junit4]   2> 219671 T419 C:control_collection S:shard1 C63 P31580 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 219671 T419 C:control_collection S:shard1 C63 P31580 oasc.SyncStrategy.syncToMe http://127.0.0.1:31580/_/h/collection1/ has no replicas
   [junit4]   2> 219672 T412 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 219672 T423 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@78d53fb8 name:ZooKeeperConnection Watcher:127.0.0.1:25045/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 219672 T419 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:31580/_/h/collection1/ shard1
   [junit4]   2> 219672 T385 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 219673 T419 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 219673 T411 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 219673 T385 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 219674 T385 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 219678 T411 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 219678 T385 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 219679 T412 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:31580/_/h",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 219782 T411 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 219782 T423 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 219830 T419 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:31580/_/h/collection1/ and leader is http://127.0.0.1:31580/_/h/collection1/
   [junit4]   2> 219831 T419 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:31580/_/h
   [junit4]   2> 219831 T419 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 219832 T419 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 219832 T419 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 219834 T411 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 219836 T412 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:31580/_/h",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:31580__%2Fh",
   [junit4]   2> 	  "core_node_name":"core_node1"} current state version: 3
   [junit4]   2> 219837 T412 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:31580/_/h",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:31580__%2Fh",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 220000 T423 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 220000 T411 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 222392 T385 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/cores/collection1
   [junit4]   2> 222397 T385 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001
   [junit4]   2> 222399 T385 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 222403 T385 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:59242
   [junit4]   2> 222403 T385 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/_/h, solrconfig=solrconfig.xml, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/tempDir-001/jetty1, hostPort=49825, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/cores}
   [junit4]   2> 222404 T385 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 222404 T385 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/'
   [junit4]   2> 222483 T385 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/solr.xml
   [junit4]   2> 222583 T385 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/cores
   [junit4]   2> 222584 T385 oasc.CoreContainer.<init> New CoreContainer 1298212678
   [junit4]   2> 222584 T385 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/]
   [junit4]   2> 222585 T385 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/lib
   [junit4]   2> 222586 T385 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/lib).
   [junit4]   2> 222602 T385 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 222603 T385 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 222603 T385 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 222604 T385 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 222604 T385 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 222604 T385 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 222605 T385 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 222605 T385 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 222606 T385 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 222606 T385 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 222606 T385 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 222630 T385 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 222632 T385 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 222632 T385 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 222633 T385 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 222633 T385 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:25045/solr
   [junit4]   2> 222633 T385 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 222634 T385 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 222635 T385 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 222637 T435 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1962080 name:ZooKeeperConnection Watcher:127.0.0.1:25045 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 222638 T385 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 222639 T385 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 222642 T385 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 222643 T438 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@373bc8de name:ZooKeeperConnection Watcher:127.0.0.1:25045/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 222644 T385 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 222649 T385 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 223654 T385 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49825__%2Fh
   [junit4]   2> 223655 T385 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49825__%2Fh
   [junit4]   2> 223658 T385 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 223661 T385 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/cores
   [junit4]   2> 223664 T385 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {transient=false, name=collection1, coreNodeName=, config=solrconfig.xml, collection=collection1, schema=schema.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/cores/collection1/, loadOnStartup=true, shard=, dataDir=data/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/cores/collection1}
   [junit4]   2> 223665 T385 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/cores/collection1/
   [junit4]   2> 223665 T385 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 223667 T439 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 223667 T439 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 223668 T439 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 223668 T411 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 223670 T412 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49825/_/h",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:49825__%2Fh"} current state version: 4
   [junit4]   2> 223670 T412 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49825/_/h",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:49825__%2Fh"}
   [junit4]   2> 223671 T412 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 223671 T412 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 223673 T438 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 223673 T423 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 223673 T411 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 224669 T439 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 224670 T439 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 224671 T439 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 224671 T439 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 224672 T439 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 224672 T439 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/cores/collection1/'
   [junit4]   2> 224697 T439 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 224700 T439 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 224717 T439 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 224761 T439 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 224842 T439 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 224843 T439 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 224859 T439 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 225486 T439 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 225511 T439 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 225515 T439 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 225530 T439 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 225537 T439 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 225542 T439 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 225543 T439 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 225544 T439 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 225544 T439 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 225545 T439 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 225545 T439 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 225546 T439 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 225546 T439 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 225546 T439 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 225547 T439 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@62c5dc6a
   [junit4]   2> 225548 T439 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/cores/collection1/data
   [junit4]   2> 225548 T439 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 225549 T439 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 225549 T439 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 225550 T439 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=24, maxMergeAtOnceExplicit=29, maxMergedSegmentMB=3.1484375, floorSegmentMB=1.59375, forceMergeDeletesPctAllowed=20.062224976109842, segmentsPerTier=43.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 225551 T439 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6ab86bf4 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@71b796e8),segFN=segments_1,generation=1}
   [junit4]   2> 225551 T439 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 225562 T439 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 225562 T439 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 225563 T439 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 225563 T439 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 225563 T439 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 225564 T439 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 225564 T439 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 225565 T439 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 225565 T439 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 225608 T439 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 225612 T439 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 225617 T439 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 225622 T439 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 225635 T439 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /config,/admin/plugins,/get,/update/json/docs,/admin/ping,/update/csv,/schema,/admin/segments,/admin/system,/admin/file,/admin/mbeans,/admin/luke,standard,/admin/logging,/admin/threads,/update,/replication,/admin/properties,/update/json
   [junit4]   2> 225638 T439 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 225641 T439 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 225641 T439 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 225644 T439 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 225644 T439 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 225645 T439 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=37, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 225646 T439 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6ab86bf4 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@71b796e8),segFN=segments_1,generation=1}
   [junit4]   2> 225647 T439 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 225647 T439 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4be308fe[collection1] main
   [junit4]   2> 225648 T439 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 225649 T439 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 225649 T439 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 225650 T439 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 225650 T439 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 225650 T439 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 225651 T439 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 225651 T439 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 225652 T439 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 225652 T439 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 225654 T440 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4be308fe[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 225655 T439 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 225656 T443 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:49825/_/h collection:collection1 shard:shard1
   [junit4]   2> 225657 T385 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 225657 T385 oass.SolrDispatchFilter.init So

[...truncated too long message...]

it4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1940494 T10737 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:57116/
   [junit4]   2> 1940494 T10737 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:57112/
   [junit4]   2> 1940494 T10737 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:57107/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=E317CCBEAFC08265 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=it_IT -Dtests.timezone=America/Toronto -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   67.9s J3 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:57112 failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([E317CCBEAFC08265:6B43F364013CEF9D]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:57112 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1940511 T10737 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest E317CCBEAFC08265-001
   [junit4]   2> 67949 T10736 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {id=PostingsFormat(name=Asserting), a_t=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), _version_=PostingsFormat(name=Memory doPackFST= true), range_facet_l=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), text=PostingsFormat(name=LuceneVarGapDocFreqInterval), timestamp=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), multiDefault=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), intDefault=PostingsFormat(name=Memory doPackFST= true)}, docValues:{timestamp=DocValuesFormat(name=Memory)}, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=it_IT, timezone=America/Toronto
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=103991192,total=342884352
   [junit4]   2> NOTE: All tests run in this JVM: [AddSchemaFieldsUpdateProcessorFactoryTest, TestSimpleTrackingShardHandler, TermsComponentTest, TestPerFieldSimilarity, TestBM25SimilarityFactory, TestImplicitCoreProperties, DocValuesMissingTest, TestDefaultSearchFieldResource, CSVRequestHandlerTest, SortByFunctionTest, CoreAdminRequestStatusTest, MergeStrategyTest, TestFastOutputStream, OpenCloseCoreStressTest, TestPivotHelperCode, RequestLoggingTest, UniqFieldsUpdateProcessorFactoryTest, RequestHandlersTest, StatelessScriptUpdateProcessorFactoryTest, TestStandardQParsers, TestShardHandlerFactory, SchemaVersionSpecificBehaviorTest, TestConfigOverlay, TestFieldSortValues, TestReloadAndDeleteDocs, TestSolrConfigHandler, AtomicUpdatesTest, DistribCursorPagingTest, SignatureUpdateProcessorFactoryTest, TestReRankQParserPlugin, TestBinaryField, TestReversedWildcardFilterFactory, CollectionsAPIDistributedZkTest, DistributedQueryElevationComponentTest, DocValuesTest, SuggesterTest, LeaderInitiatedRecoveryOnCommitTest, TestConfigSets, FullSolrCloudDistribCmdsTest, TestUpdate, ZkSolrClientTest, TestManagedSchema, FieldAnalysisRequestHandlerTest, TestCustomSort, ExternalCollectionsTest, SharedFSAutoReplicaFailoverUtilsTest, TestManagedStopFilterFactory, TestMacros, OutputWriterTest, DistributedMLTComponentTest, TestDynamicLoading, CoreMergeIndexesAdminHandlerTest, TestCSVResponseWriter, TestStressReorder, TestManagedSchemaDynamicFieldResource, ClusterStateUpdateTest, DirectUpdateHandlerOptimizeTest, URLClassifyProcessorTest, DistributedFacetPivotSmallTest, TestFiltering, BasicDistributedZk2Test, BasicDistributedZkTest, TestSolrConfigHandlerCloud, LeaderElectionIntegrationTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J3 in 68.48s, 1 test, 1 error <<< FAILURES!

[...truncated 709 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 4 errors, 61 ignored (21 assumptions)

Total time: 60 minutes 36 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 208692924 bytes
Compression is 0.0%
Took 43 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2886 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2886/

1 tests failed.
REGRESSION:  org.apache.lucene.analysis.uima.UIMABaseAnalyzerTest.testRandomStringsWithConfigurationParameters

Error Message:
some thread(s) failed

Stack Trace:
java.lang.RuntimeException: some thread(s) failed
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:531)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:428)
	at org.apache.lucene.analysis.uima.UIMABaseAnalyzerTest.testRandomStringsWithConfigurationParameters(UIMABaseAnalyzerTest.java:133)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 3939 lines...]
   [junit4] Suite: org.apache.lucene.analysis.uima.UIMABaseAnalyzerTest
   [junit4]   2> Abr 03, 2015 8:02:12 PM WhitespaceTokenizer initialize
   [junit4]   2> INFO: "Whitespace tokenizer successfully initialized"
   [junit4]   2> Abr 03, 2015 8:02:12 PM WhitespaceTokenizer typeSystemInit
   [junit4]   2> INFO: "Whitespace tokenizer typesystem initialized"
   [junit4]   2> Abr 03, 2015 8:02:12 PM WhitespaceTokenizer process
   [junit4]   2> INFO: "Whitespace tokenizer starts processing"
   [junit4]   2> Abr 03, 2015 8:02:12 PM WhitespaceTokenizer process
   [junit4]   2> INFO: "Whitespace tokenizer finished processing"
   [junit4]   2> Abr 03, 2015 8:02:12 PM WhitespaceTokenizer process
   [junit4]   2> INFO: "Whitespace tokenizer starts processing"
   [junit4]   2> Abr 03, 2015 8:02:12 PM WhitespaceTokenizer process
   [junit4]   2> INFO: "Whitespace tokenizer finished processing"
   [junit4]   2> Abr 03, 2015 8:02:13 PM WhitespaceTokenizer process
   [junit4]   2> INFO: "Whitespace tokenizer starts processing"
   [junit4]   2> Abr 03, 2015 8:02:13 PM WhitespaceTokenizer process
   [junit4]   2> INFO: "Whitespace tokenizer finished processing"
   [junit4]   2> Abr 03, 2015 8:02:13 PM WhitespaceTokenizer process
   [junit4]   2> INFO: "Whitespace tokenizer starts processing"
   [junit4]   2> Abr 03, 2015 8:02:13 PM WhitespaceTokenizer process
   [junit4]   2> INFO: "Whitespace tokenizer finished processing"
   [junit4]   2> TEST FAIL: useCharFilter=false text='yej wbw  xeln \u0e97\u0ed4\u0e80'
   [junit4]   2> Abr 03, 2015 8:02:14 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: Thread[Thread-8,5,TGRP-UIMABaseAnalyzerTest]
   [junit4]   2> java.lang.AssertionError
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([5C1B564E414DC109]:0)
   [junit4]   2> 	at sun.reflect.generics.reflectiveObjects.WildcardTypeImpl.getLowerBoundASTs(WildcardTypeImpl.java:94)
   [junit4]   2> 	at sun.reflect.generics.reflectiveObjects.WildcardTypeImpl.getLowerBounds(WildcardTypeImpl.java:165)
   [junit4]   2> 	at com.sun.beans.TypeResolver.resolve(TypeResolver.java:176)
   [junit4]   2> 	at com.sun.beans.TypeResolver.resolve(TypeResolver.java:218)
   [junit4]   2> 	at com.sun.beans.TypeResolver.resolve(TypeResolver.java:169)
   [junit4]   2> 	at com.sun.beans.TypeResolver.resolveInClass(TypeResolver.java:81)
   [junit4]   2> 	at java.beans.FeatureDescriptor.getReturnType(FeatureDescriptor.java:370)
   [junit4]   2> 	at java.beans.Introspector.getTargetEventInfo(Introspector.java:996)
   [junit4]   2> 	at java.beans.Introspector.getBeanInfo(Introspector.java:417)
   [junit4]   2> 	at java.beans.Introspector.getBeanInfo(Introspector.java:252)
   [junit4]   2> 	at java.beans.Introspector.<init>(Introspector.java:397)
   [junit4]   2> 	at java.beans.Introspector.getBeanInfo(Introspector.java:252)
   [junit4]   2> 	at java.beans.Introspector.<init>(Introspector.java:397)
   [junit4]   2> 	at java.beans.Introspector.getBeanInfo(Introspector.java:252)
   [junit4]   2> 	at java.beans.Introspector.getBeanInfo(Introspector.java:194)
   [junit4]   2> 	at org.apache.uima.resource.metadata.impl.MetaDataObject_impl.getPropertyDescriptors(MetaDataObject_impl.java:1269)
   [junit4]   2> 	at org.apache.uima.resource.metadata.impl.MetaDataObject_impl.listAttributes(MetaDataObject_impl.java:105)
   [junit4]   2> 	at org.apache.uima.resource.metadata.impl.MetaDataObject_impl.setSourceUrl(MetaDataObject_impl.java:289)
   [junit4]   2> 	at org.apache.uima.util.CasCreationUtils.addTypeToMergedTypeSystem(CasCreationUtils.java:1125)
   [junit4]   2> 	at org.apache.uima.util.CasCreationUtils.mergeTypeSystems(CasCreationUtils.java:1090)
   [junit4]   2> 	at org.apache.uima.util.CasCreationUtils.mergeTypeSystems(CasCreationUtils.java:1019)
   [junit4]   2> 	at org.apache.uima.resource.CasDefinition.<init>(CasDefinition.java:78)
   [junit4]   2> 	at org.apache.uima.resource.impl.CasManager_impl.getCasDefinition(CasManager_impl.java:94)
   [junit4]   2> 	at org.apache.uima.analysis_engine.impl.AnalysisEngineImplBase.newCAS(AnalysisEngineImplBase.java:230)
   [junit4]   2> 	at org.apache.lucene.analysis.uima.BaseUIMATokenizer.analyzeInput(BaseUIMATokenizer.java:67)
   [junit4]   2> 	at org.apache.lucene.analysis.uima.UIMAAnnotationsTokenizer.initializeIterator(UIMAAnnotationsTokenizer.java:60)
   [junit4]   2> 	at org.apache.lucene.analysis.uima.UIMAAnnotationsTokenizer.incrementToken(UIMAAnnotationsTokenizer.java:72)
   [junit4]   2> 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkAnalysisConsistency(BaseTokenStreamTestCase.java:700)
   [junit4]   2> 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:611)
   [junit4]   2> 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.access$000(BaseTokenStreamTestCase.java:60)
   [junit4]   2> 	at org.apache.lucene.analysis.BaseTokenStreamTestCase$AnalysisThread.run(BaseTokenStreamTestCase.java:479)
   [junit4]   2> 
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=UIMABaseAnalyzerTest -Dtests.method=testRandomStringsWithConfigurationParameters -Dtests.seed=5C1B564E414DC109 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=pt_PT -Dtests.timezone=SystemV/AST4ADT -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   1.52s | UIMABaseAnalyzerTest.testRandomStringsWithConfigurationParameters <<<
   [junit4]    > Throwable #1: java.lang.RuntimeException: some thread(s) failed
   [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:531)
   [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:428)
   [junit4]    > 	at org.apache.lucene.analysis.uima.UIMABaseAnalyzerTest.testRandomStringsWithConfigurationParameters(UIMABaseAnalyzerTest.java:133)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #2: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=19, name=Thread-8, state=RUNNABLE, group=TGRP-UIMABaseAnalyzerTest]
   [junit4]    > Caused by: java.lang.AssertionError
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([5C1B564E414DC109]:0)
   [junit4]    > 	at com.sun.beans.TypeResolver.resolve(TypeResolver.java:176)
   [junit4]    > 	at com.sun.beans.TypeResolver.resolve(TypeResolver.java:218)
   [junit4]    > 	at com.sun.beans.TypeResolver.resolve(TypeResolver.java:169)
   [junit4]    > 	at com.sun.beans.TypeResolver.resolveInClass(TypeResolver.java:81)
   [junit4]    > 	at java.beans.FeatureDescriptor.getReturnType(FeatureDescriptor.java:370)
   [junit4]    > 	at java.beans.Introspector.getTargetEventInfo(Introspector.java:996)
   [junit4]    > 	at java.beans.Introspector.getBeanInfo(Introspector.java:417)
   [junit4]    > 	at java.beans.Introspector.getBeanInfo(Introspector.java:252)
   [junit4]    > 	at java.beans.Introspector.<init>(Introspector.java:397)
   [junit4]    > 	at java.beans.Introspector.getBeanInfo(Introspector.java:252)
   [junit4]    > 	at java.beans.Introspector.<init>(Introspector.java:397)
   [junit4]    > 	at java.beans.Introspector.getBeanInfo(Introspector.java:252)
   [junit4]    > 	at java.beans.Introspector.getBeanInfo(Introspector.java:194)
   [junit4]    > 	at org.apache.uima.resource.metadata.impl.MetaDataObject_impl.getPropertyDescriptors(MetaDataObject_impl.java:1269)
   [junit4]    > 	at org.apache.uima.resource.metadata.impl.MetaDataObject_impl.listAttributes(MetaDataObject_impl.java:105)
   [junit4]    > 	at org.apache.uima.resource.metadata.impl.MetaDataObject_impl.setSourceUrl(MetaDataObject_impl.java:289)
   [junit4]    > 	at org.apache.uima.util.CasCreationUtils.addTypeToMergedTypeSystem(CasCreationUtils.java:1125)
   [junit4]    > 	at org.apache.uima.util.CasCreationUtils.mergeTypeSystems(CasCreationUtils.java:1090)
   [junit4]    > 	at org.apache.uima.util.CasCreationUtils.mergeTypeSystems(CasCreationUtils.java:1019)
   [junit4]    > 	at org.apache.uima.resource.CasDefinition.<init>(CasDefinition.java:78)
   [junit4]    > 	at org.apache.uima.resource.impl.CasManager_impl.getCasDefinition(CasManager_impl.java:94)
   [junit4]    > 	at org.apache.uima.analysis_engine.impl.AnalysisEngineImplBase.newCAS(AnalysisEngineImplBase.java:230)
   [junit4]    > 	at org.apache.lucene.analysis.uima.BaseUIMATokenizer.analyzeInput(BaseUIMATokenizer.java:67)
   [junit4]    > 	at org.apache.lucene.analysis.uima.UIMAAnnotationsTokenizer.initializeIterator(UIMAAnnotationsTokenizer.java:60)
   [junit4]    > 	at org.apache.lucene.analysis.uima.UIMAAnnotationsTokenizer.incrementToken(UIMAAnnotationsTokenizer.java:72)
   [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkAnalysisConsistency(BaseTokenStreamTestCase.java:700)
   [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:611)
   [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.access$000(BaseTokenStreamTestCase.java:60)
   [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase$AnalysisThread.run(BaseTokenStreamTestCase.java:479)
   [junit4]   2> Abr 03, 2015 8:02:18 PM WhitespaceTokenizer initialize
   [junit4]   2> INFO: "Whitespace tokenizer successfully initialized"
   [junit4]   2> Abr 03, 2015 8:02:18 PM WhitespaceTokenizer typeSystemInit
   [junit4]   2> INFO: "Whitespace tokenizer typesystem initialized"
   [junit4]   2> Abr 03, 2015 8:02:18 PM WhitespaceTokenizer process
   [junit4]   2> INFO: "Whitespace tokenizer starts processing"
   [junit4]   2> Abr 03, 2015 8:02:18 PM WhitespaceTokenizer process
   [junit4]   2> INFO: "Whitespace tokenizer finished processing"
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {dummy=FSTOrd50, title=FSTOrd50, contents=FSTOrd50}, docValues:{}, sim=DefaultSimilarity, locale=pt_PT, timezone=SystemV/AST4ADT
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=122198424,total=301465600
   [junit4]   2> NOTE: All tests run in this JVM: [UIMABaseAnalyzerTest]
   [junit4] Completed in 11.63s, 4 tests, 1 error <<< FAILURES!

[...truncated 18 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/build.xml:456: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:2166: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/analysis/build.xml:106: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/analysis/build.xml:38: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/module-build.xml:58: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 5 suites, 12 tests, 1 error

Total time: 8 minutes 20 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 2 artifacts
Archive block size is 32768
Received 0 blocks and 54261 bytes
Compression is 0.0%
Took 0.4 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2885 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2885/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:16808/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16808/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([8632F28962AA5D4B:E66CD53CC5630B3]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:16808 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:65051/jn_qr/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:65051/jn_qr/collection1
	at __randomizedtesting.SeedInfo.seed([8632F28962AA5D4B:E66CD53CC5630B3]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:65051/jn_qr/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:65051 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:10434/i/p/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:10434/i/p/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([8632F28962AA5D4B:E66CD53CC5630B3]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:10434/i/p/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:10434 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9407 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/init-core-data-001
   [junit4]   2> 190587 T677 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /jn_qr/
   [junit4]   2> 190592 T677 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 190593 T678 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 190693 T677 oasc.ZkTestServer.run start zk server on port:65022
   [junit4]   2> 190694 T677 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 190695 T677 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 190699 T685 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@302109de name:ZooKeeperConnection Watcher:127.0.0.1:65022 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 190699 T677 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 190700 T677 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 190700 T677 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 190703 T677 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 190704 T677 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 190706 T688 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@75a792d8 name:ZooKeeperConnection Watcher:127.0.0.1:65022/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 190706 T677 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 190707 T677 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 190707 T677 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 190709 T677 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 190711 T677 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 190713 T677 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 190715 T677 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 190716 T677 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 190719 T677 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 190720 T677 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 190823 T677 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 190823 T677 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 190825 T677 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 190826 T677 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 190827 T677 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 190828 T677 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 190830 T677 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 190830 T677 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 190832 T677 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 190832 T677 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 190834 T677 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 190834 T677 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 190836 T677 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 190836 T677 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 190838 T677 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 190839 T677 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 190841 T677 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 190841 T677 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 192005 T677 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/cores/collection1
   [junit4]   2> 192010 T677 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 192014 T677 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:65038
   [junit4]   2> 192015 T677 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/jn_qr, hostPort=65037, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/tempDir-001/control/data}
   [junit4]   2> 192016 T677 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 192016 T677 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/'
   [junit4]   2> 192065 T677 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/solr.xml
   [junit4]   2> 192128 T677 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/cores
   [junit4]   2> 192129 T677 oasc.CoreContainer.<init> New CoreContainer 359853771
   [junit4]   2> 192129 T677 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/]
   [junit4]   2> 192130 T677 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/lib
   [junit4]   2> 192131 T677 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/lib).
   [junit4]   2> 192145 T677 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 192146 T677 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 192146 T677 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 192146 T677 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 192147 T677 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 192147 T677 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 192148 T677 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 192148 T677 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 192148 T677 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 192149 T677 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 192149 T677 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 192157 T677 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 192159 T677 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 192159 T677 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 192159 T677 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 192160 T677 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:65022/solr
   [junit4]   2> 192161 T677 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 192161 T677 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 192162 T677 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 192164 T700 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4da90726 name:ZooKeeperConnection Watcher:127.0.0.1:65022 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 192164 T677 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 192165 T677 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 192167 T677 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 192169 T703 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@470d25cb name:ZooKeeperConnection Watcher:127.0.0.1:65022/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 192169 T677 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 192171 T677 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 192174 T677 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 192176 T677 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 192179 T677 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 192182 T677 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 192184 T677 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 192186 T677 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 192188 T677 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 192190 T677 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:65037_jn_qr
   [junit4]   2> 192191 T677 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:65037_jn_qr
   [junit4]   2> 192193 T677 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 192194 T677 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 192196 T677 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 192198 T677 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:65037_jn_qr
   [junit4]   2> 192198 T677 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 192200 T677 oasc.Overseer.start Overseer (id=93591450468089859-127.0.0.1:65037_jn_qr-n_0000000000) starting
   [junit4]   2> 192202 T677 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 192210 T677 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 192211 T705 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 192211 T677 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 192211 T704 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 192216 T677 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/cores
   [junit4]   2> 192218 T677 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, config=solrconfig.xml, collection=control_collection, schema=schema.xml, dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/cores/collection1/, transient=false, loadOnStartup=true, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/cores/collection1, shard=, coreNodeName=}
   [junit4]   2> 192219 T677 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/cores/collection1/
   [junit4]   2> 192219 T677 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 192221 T707 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 192221 T707 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 192222 T703 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 192222 T707 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 192223 T704 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:65037/jn_qr",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:65037_jn_qr",
   [junit4]   2> 	  "roles":null} current state version: 0
   [junit4]   2> 192224 T704 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:65037/jn_qr",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:65037_jn_qr",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 192224 T704 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 192225 T704 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 192226 T703 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 193223 T707 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 193224 T707 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 193225 T707 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 193225 T707 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 193225 T707 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 193226 T707 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/cores/collection1/'
   [junit4]   2> 193240 T707 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 193242 T707 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 193251 T707 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 193271 T707 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 193310 T707 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 193411 T707 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 193421 T707 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 193988 T707 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 194002 T707 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 194004 T707 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 194065 T707 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 194070 T707 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 194075 T707 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 194076 T707 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 194076 T707 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 194076 T707 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 194077 T707 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 194077 T707 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 194078 T707 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 194078 T707 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 194078 T707 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/cores/collection1/data/
   [junit4]   2> 194079 T707 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@118c55d6
   [junit4]   2> 194079 T707 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/cores/collection1/data
   [junit4]   2> 194080 T707 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/cores/collection1/data/index/
   [junit4]   2> 194080 T707 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 194080 T707 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/cores/collection1/data/index
   [junit4]   2> 194081 T707 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=11, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 194081 T707 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@41dc3a24 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@616d3343),segFN=segments_1,generation=1}
   [junit4]   2> 194082 T707 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 194087 T707 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 194088 T707 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 194088 T707 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 194088 T707 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 194088 T707 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 194089 T707 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 194089 T707 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 194089 T707 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 194089 T707 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 194110 T707 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 194113 T707 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 194115 T707 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 194117 T707 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 194123 T707 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/plugins,/admin/system,/replication,/admin/ping,/admin/segments,/update/csv,/schema,/admin/properties,/admin/threads,/config,/update/json/docs,/admin/file,standard,/admin/logging,/update/json,/update,/admin/luke,/admin/mbeans,/get
   [junit4]   2> 194127 T707 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 194128 T707 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 194129 T707 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 194130 T707 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 194130 T707 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 194131 T707 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=29, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=59.6513671875, floorSegmentMB=1.1591796875, forceMergeDeletesPctAllowed=20.822084393283095, segmentsPerTier=50.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 194131 T707 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@41dc3a24 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@616d3343),segFN=segments_1,generation=1}
   [junit4]   2> 194132 T707 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 194132 T707 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4d36517d[collection1] main
   [junit4]   2> 194132 T707 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 194133 T707 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 194133 T707 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 194134 T707 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 194134 T707 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 194134 T707 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 194135 T707 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 194135 T707 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 194135 T707 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 194136 T707 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 194136 T708 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4d36517d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 194137 T707 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 194138 T711 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:65037/jn_qr collection:control_collection shard:shard1
   [junit4]   2> 194138 T677 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 194138 T677 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 194139 T711 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 194142 T677 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 194143 T677 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 194143 T711 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 194144 T715 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2f3e08d5 name:ZooKeeperConnection Watcher:127.0.0.1:65022/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 194145 T677 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 194145 T677 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 194145 T703 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 194145 T711 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 194146 T711 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 194145 T677 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 194146 T704 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C50 name=collection1 org.apache.solr.core.SolrCore@79ed71bd url=http://127.0.0.1:65037/jn_qr/collection1 node=127.0.0.1:65037_jn_qr C50_STATE=coll:control_collection core:collection1 props:{state=down, core=collection1, base_url=http://127.0.0.1:65037/jn_qr, node_name=127.0.0.1:65037_jn_qr}
   [junit4]   2> 194146 T711 C:control_collection S:shard1 C50 P65037 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:65037/jn_qr/collection1/
   [junit4]   2> 194147 T711 C:control_collection S:shard1 C50 P65037 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 194147 T711 C:control_collection S:shard1 C50 P65037 oasc.SyncStrategy.syncToMe http://127.0.0.1:65037/jn_qr/collection1/ has no replicas
   [junit4]   2> 194147 T711 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:65037/jn_qr/collection1/ shard1
   [junit4]   2> 194147 T711 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 194148 T715 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 194148 T703 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 194150 T677 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 194152 T703 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 194153 T704 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:65037/jn_qr",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 194256 T715 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 194256 T703 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 194305 T711 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:65037/jn_qr/collection1/ and leader is http://127.0.0.1:65037/jn_qr/collection1/
   [junit4]   2> 194306 T711 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:65037/jn_qr
   [junit4]   2> 194306 T711 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 194306 T711 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 194307 T711 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 194308 T703 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 194310 T704 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:65037/jn_qr",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:65037_jn_qr",
   [junit4]   2> 	  "roles":null} current state version: 3
   [junit4]   2> 194310 T704 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:65037/jn_qr",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:65037_jn_qr",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 194414 T715 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 194414 T703 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 195467 T677 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/cores/collection1
   [junit4]   2> 195469 T677 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001
   [junit4]   2> 195470 T677 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 195472 T677 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:65052
   [junit4]   2> 195473 T677 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/jn_qr, hostPort=65051, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/cores}
   [junit4]   2> 195473 T677 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 195473 T677 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/'
   [junit4]   2> 195509 T677 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/solr.xml
   [junit4]   2> 195547 T677 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/cores
   [junit4]   2> 195547 T677 oasc.CoreContainer.<init> New CoreContainer 670326152
   [junit4]   2> 195548 T677 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/]
   [junit4]   2> 195548 T677 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/lib
   [junit4]   2> 195548 T677 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/lib).
   [junit4]   2> 195558 T677 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 195559 T677 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 195559 T677 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 195559 T677 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 195559 T677 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 195560 T677 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 195560 T677 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 195560 T677 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 195561 T677 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 195561 T677 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 195561 T677 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 195566 T677 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 195567 T677 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 195568 T677 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 195568 T677 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 195568 T677 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:65022/solr
   [junit4]   2> 195568 T677 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 195569 T677 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 195569 T677 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 195571 T727 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5ca40bc7 name:ZooKeeperConnection Watcher:127.0.0.1:65022 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 195572 T677 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 195573 T677 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 195574 T677 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 195576 T730 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@360d8c9a name:ZooKeeperConnection Watcher:127.0.0.1:65022/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 195576 T677 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 195582 T677 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 196587 T677 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:65051_jn_qr
   [junit4]   2> 196588 T677 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:65051_jn_qr
   [junit4]   2> 196592 T677 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 196595 T677 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/cores
   [junit4]   2> 196598 T677 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {schema=schema.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/cores/collection1/, transient=false, config=solrconfig.xml, shard=, dataDir=data/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/cores/collection1, collection=collection1, name=collection1, coreNodeName=, loadOnStartup=true}
   [junit4]   2> 196598 T677 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/cores/collection1/
   [junit4]   2> 196599 T677 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 196600 T731 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 196601 T731 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 196602 T703 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 196602 T731 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 196603 T704 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:65051/jn_qr",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:65051_jn_qr",
   [junit4]   2> 	  "roles":null} current state version: 4
   [junit4]   2> 196604 T704 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:65051/jn_qr",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:65051_jn_qr",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 196604 T704 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 196604 T704 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 196606 T730 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 196606 T703 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 196606 T715 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 197603 T731 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 197604 T731 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 197604 T731 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 197604 T731 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 197605 T731 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 197606 T731 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/cores/collection1/'
   [junit4]   2> 197621 T731 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 197624 T731 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 197635 T731 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 197657 T731 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 197703 T731 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 197805 T731 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 197816 T731 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 198272 T731 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 198293 T731 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 198297 T731 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 198318 T731 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 198324 T731 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 198330 T731 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 198331 T731 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 198331 T731 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 198332 T731 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 198333 T731 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 198333 T731 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 198334 T731 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 198334 T731 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 198334 T731 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 198335 T731 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@118c55d6
   [junit4]   2> 198336 T731 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/cores/collection1/data
   [junit4]   2> 198336 T731 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 198337 T731 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 198337 T731 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 198338 T731 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=11, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 198339 T731 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3e14f86f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@52c25f40),segFN=segments_1,generation=1}
   [junit4]   2> 198339 T731 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 198350 T731 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 198351 T731 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 198351 T731 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 198352 T731 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 198352 T731 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 198352 T731 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 198354 T731 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 198354 T731 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 198354 T731 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 198389 T731 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 198392 T731 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 198395 T731 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 198398 T731 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 198408 T731 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/plugins,/admin/system,/replication,/admin/ping,/admin/segments,/update/csv,/schema,/admin/properties,/admin/threads,/config,/update/json/docs,/admin/file,standard,/admin/logging,/update/json,/update,/admin/luke,/admin/mbeans,/get
   [junit4]   2> 198411 T731 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 198413 T731 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 198414 T731 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 198416 T731 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 198416 T731 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 198417 T731 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=29, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=59.6513671875, floorSegmentMB=1.1591796875, forceMergeDeletesPctAllowed=20.822084393283095, segmentsPerTier=50.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 198418 T731 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3e14f86f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@52c25f40),segFN=segments_1,generation=1}
   [junit4]   2> 198418 T731 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 198419 T731 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@143272e2[collection1] main
   [junit4]   2> 198419 T731 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 198420 T731 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 198421 T731 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 198421 T731 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 198422 T731 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 198422 T731 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 198423 T731 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 198423 T731 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 198423 T731 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 198424 T731 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 198425 T732 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@143272e2[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 198426 T731 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 198427 T735 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:65051/jn_qr collection:collection1 shard:shard1
   [junit4]   2> 198427 T677 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 198428 T677 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 198428 T735 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 198435 T735 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 198437 T703 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 198437 T735 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 198438 T735 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C51 name=collection1 org.apache.solr.core.SolrCore@5fa8113e url=http://127.0.0.1:65051/jn_qr/collection1 node=127.0.0.1:65051_jn_qr C51_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, base_url=http://127.0.0.1:65051/jn_qr, node_name=127.0.0.1:65051_jn_qr}
   [junit4]   2> 198438 T735 C:collection1 S:shard1 C51 P65051 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:65051/jn_qr/collection1/
   [junit4]   2> 198439 T704 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 198439 T735 C:collection1 S:shard1 C51 P65051 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 198439 T735 C:collection1 S:shard1 C51 P65051 oasc.SyncStrategy.syncToMe http://127.0.0.1:65051/jn_qr/collection1/ has no replicas
   [junit4]   2> 198440 T735 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:65051/jn_qr/collection1/ shard1
   [junit4]   2> 198440 T735 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 198441 T703 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 198441 T730 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 198441 T715 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 198446 T703 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 198447 T704 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:65051/jn_qr",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 198552 T703 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 198552 T730 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 198552 T715 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 198599 T735 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:65051/jn_qr/collection1/ and leader is http://127.0.0.1:65051/jn_qr/collection1/
   [junit4]   2> 198600 T735 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:65051/jn_qr
   [junit4]   2> 198600 T735 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 198601 T735 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 198601 T735 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 198603 T703 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 198604 T704 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:65051/jn_qr",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:65051_jn_qr",
   [junit4]   2> 	  "roles":null} current state version: 7
   [junit4]   2> 198605 T704 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:65051/jn_qr",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:65051_jn_qr",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 198710 T703 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 198710 T715 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 198710 T730 oascc.ZkSta

[...truncated too long message...]

unit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent data watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		2	/solr/collections/c8n_1x3_commits/leader_elect/shard1/election/93591532682346506-core_node2-n_0000000000
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1524390 T9440 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16818/
   [junit4]   2> 1524390 T9440 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16812/
   [junit4]   2> 1524391 T9440 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 13 connections to: http://127.0.0.1:16808/
   [junit4]   2> 1524391 T9440 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16832/
   [junit4]   2> 1524391 T9440 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 3 connections to: http://127.0.0.1:16800/
   [junit4]   2> 1524391 T9440 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16818/
   [junit4]   2> 1524392 T9440 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16812/
   [junit4]   2> 1524392 T9440 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16808/
   [junit4]   2> 1524392 T9440 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16832/
   [junit4]   2> 1524392 T9440 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16800/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=8632F28962AA5D4B -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=fi -Dtests.timezone=America/Argentina/Cordoba -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   79.1s J2 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16808/c8n_1x3_commits_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([8632F28962AA5D4B:E66CD53CC5630B3]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:16808 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 48 more
   [junit4]   2> 1524405 T9440 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8632F28962AA5D4B-001
   [junit4]   2> 79140 T9439 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=DefaultSimilarity, locale=fi, timezone=America/Argentina/Cordoba
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=181910080,total=352321536
   [junit4]   2> NOTE: All tests run in this JVM: [TestSolrQueryParserResource, ChangedSchemaMergeTest, BinaryUpdateRequestHandlerTest, RecoveryZkTest, UnloadDistributedZkTest, HdfsWriteToMultipleCollectionsTest, TermVectorComponentDistributedTest, ExternalFileFieldSortTest, SolrInfoMBeanTest, TestLRUCache, TestCopyFieldCollectionResource, XsltUpdateRequestHandlerTest, DistributedQueueTest, ScriptEngineTest, CachingDirectoryFactoryTest, TestReplicationHandler, PreAnalyzedUpdateProcessorTest, FullSolrCloudDistribCmdsTest, SchemaVersionSpecificBehaviorTest, ActionThrottleTest, TestRestManager, TestFastOutputStream, SharedFSAutoReplicaFailoverUtilsTest, SuggestComponentTest, QueryEqualityTest, TestSolrQueryParser, LeaderElectionIntegrationTest, DistributedDebugComponentTest, TestSimpleTrackingShardHandler, BJQParserTest, DistributedQueryComponentCustomSortTest, TestShardHandlerFactory, TestFaceting, TestSolrDeletionPolicy1, PolyFieldTest, AliasIntegrationTest, JSONWriterTest, HighlighterConfigTest, TestMacros, TestCollationField, PrimUtilsTest, SolrCoreCheckLockOnStartupTest, DistanceUnitsTest, SpellingQueryConverterTest, TestRequestStatusCollectionAPI, TestRangeQuery, TestStressReorder, TestInitParams, DistanceFunctionTest, TestOmitPositions, TestCryptoKeys, SyncSliceTest, CacheHeaderTest, MigrateRouteKeyTest, TestDynamicLoading, TestReload, SolrCmdDistributorTest, TestCSVResponseWriter, HdfsBasicDistributedZkTest, TestSolrConfigHandlerCloud, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J2 in 79.93s, 1 test, 1 error <<< FAILURES!

[...truncated 742 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 53 minutes 54 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 214079855 bytes
Compression is 0.0%
Took 41 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2884 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2884/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:20844/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:20844/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([691015E6B7971E47:E1442A3C196B73BF]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:20844 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:21221/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:21221/collection1
	at __randomizedtesting.SeedInfo.seed([691015E6B7971E47:E1442A3C196B73BF]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:21221/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:21221 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:21032/pv/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:21032/pv/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([691015E6B7971E47:E1442A3C196B73BF]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:21032/pv/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:21032 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 10033 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/init-core-data-001
   [junit4]   2> 1450379 T4115 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 1450384 T4115 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1450384 T4116 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1450484 T4115 oasc.ZkTestServer.run start zk server on port:20824
   [junit4]   2> 1450485 T4115 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1450485 T4115 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1450488 T4123 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@73910be7 name:ZooKeeperConnection Watcher:127.0.0.1:20824 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1450489 T4115 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1450489 T4115 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1450489 T4115 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1450492 T4115 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1450492 T4115 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1450493 T4126 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@70fb2281 name:ZooKeeperConnection Watcher:127.0.0.1:20824/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1450493 T4115 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1450494 T4115 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1450494 T4115 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1450495 T4115 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1450496 T4115 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1450497 T4115 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1450499 T4115 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1450499 T4115 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1450501 T4115 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1450502 T4115 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1450604 T4115 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1450604 T4115 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1450606 T4115 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1450606 T4115 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1450607 T4115 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1450608 T4115 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1450609 T4115 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1450610 T4115 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1450611 T4115 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1450611 T4115 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1450613 T4115 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1450613 T4115 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1450615 T4115 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1450615 T4115 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1450616 T4115 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1450617 T4115 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1450618 T4115 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1450619 T4115 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1452445 T4115 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/cores/collection1
   [junit4]   2> 1452450 T4115 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1452452 T4115 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:20834
   [junit4]   2> 1452453 T4115 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/cores, hostContext=/, hostPort=20833, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/tempDir-001/control/data}
   [junit4]   2> 1452453 T4115 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 1452453 T4115 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/'
   [junit4]   2> 1452496 T4115 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/solr.xml
   [junit4]   2> 1452549 T4115 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/cores
   [junit4]   2> 1452550 T4115 oasc.CoreContainer.<init> New CoreContainer 1479276749
   [junit4]   2> 1452550 T4115 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/]
   [junit4]   2> 1452550 T4115 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/lib
   [junit4]   2> 1452551 T4115 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/lib).
   [junit4]   2> 1452560 T4115 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1452560 T4115 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1452561 T4115 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1452561 T4115 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1452561 T4115 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1452561 T4115 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1452561 T4115 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1452562 T4115 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1452562 T4115 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1452562 T4115 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1452562 T4115 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1452569 T4115 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1452570 T4115 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1452570 T4115 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1452571 T4115 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1452571 T4115 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:20824/solr
   [junit4]   2> 1452571 T4115 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1452571 T4115 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1452572 T4115 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1452573 T4138 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@12323214 name:ZooKeeperConnection Watcher:127.0.0.1:20824 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1452574 T4115 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1452575 T4115 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1452576 T4115 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1452577 T4141 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@52569ce1 name:ZooKeeperConnection Watcher:127.0.0.1:20824/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1452578 T4115 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1452579 T4115 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1452580 T4115 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1452581 T4115 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1452582 T4115 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1452583 T4115 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1452585 T4115 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1452586 T4115 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1452587 T4115 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1452587 T4115 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:20833_
   [junit4]   2> 1452588 T4115 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:20833_
   [junit4]   2> 1452589 T4115 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1452590 T4115 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1452590 T4115 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1452591 T4115 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:20833_
   [junit4]   2> 1452592 T4115 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1452592 T4115 oasc.Overseer.start Overseer (id=93590608661118979-127.0.0.1:20833_-n_0000000000) starting
   [junit4]   2> 1452594 T4115 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1452597 T4115 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1452598 T4143 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1452598 T4115 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1452598 T4142 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1452601 T4115 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/cores
   [junit4]   2> 1452603 T4115 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/cores/collection1, schema=schema.xml, coreNodeName=, collection=control_collection, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/cores/collection1/, config=solrconfig.xml, loadOnStartup=true, name=collection1, dataDir=data/, transient=false}
   [junit4]   2> 1452603 T4115 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/cores/collection1/
   [junit4]   2> 1452603 T4115 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1452604 T4145 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1452604 T4145 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1452605 T4141 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1452605 T4145 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1452605 T4142 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20833",
   [junit4]   2> 	  "node_name":"127.0.0.1:20833_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 0
   [junit4]   2> 1452605 T4142 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20833",
   [junit4]   2> 	  "node_name":"127.0.0.1:20833_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 1452606 T4142 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1452606 T4142 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1452607 T4141 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1453605 T4145 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1453606 T4145 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1453606 T4145 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1453607 T4145 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1453607 T4145 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1453607 T4145 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/cores/collection1/'
   [junit4]   2> 1453626 T4145 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1453628 T4145 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1453639 T4145 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1453662 T4145 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1453712 T4145 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1453814 T4145 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1453826 T4145 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1454262 T4145 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1454279 T4145 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1454282 T4145 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1454290 T4145 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1454295 T4145 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1454299 T4145 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1454299 T4145 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1454299 T4145 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1454300 T4145 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1454300 T4145 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1454300 T4145 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1454301 T4145 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1454301 T4145 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1454301 T4145 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/cores/collection1/data/
   [junit4]   2> 1454302 T4145 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5fa863b5
   [junit4]   2> 1454302 T4145 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/cores/collection1/data
   [junit4]   2> 1454303 T4145 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/cores/collection1/data/index/
   [junit4]   2> 1454303 T4145 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1454303 T4145 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/cores/collection1/data/index
   [junit4]   2> 1454304 T4145 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=10, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.15179102665711916]
   [junit4]   2> 1454305 T4145 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7188116d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1c304556),segFN=segments_1,generation=1}
   [junit4]   2> 1454305 T4145 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1454310 T4145 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1454311 T4145 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1454311 T4145 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1454311 T4145 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1454311 T4145 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1454312 T4145 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1454312 T4145 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1454312 T4145 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1454313 T4145 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1454336 T4145 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1454338 T4145 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1454341 T4145 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1454343 T4145 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1454348 T4145 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/system,/admin/segments,/admin/file,/admin/mbeans,standard,/admin/properties,/admin/threads,/update/json/docs,/admin/luke,/get,/config,/update/csv,/update/json,/admin/ping,/update,/schema,/replication,/admin/logging,/admin/plugins
   [junit4]   2> 1454350 T4145 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1454352 T4145 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1454352 T4145 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1454354 T4145 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1454354 T4145 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1454355 T4145 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=50, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5958199606090643]
   [junit4]   2> 1454355 T4145 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7188116d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1c304556),segFN=segments_1,generation=1}
   [junit4]   2> 1454355 T4145 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1454356 T4145 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@27aa81d8[collection1] main
   [junit4]   2> 1454356 T4145 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1454357 T4145 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1454357 T4145 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1454357 T4145 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1454358 T4145 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1454358 T4145 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1454358 T4145 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1454359 T4145 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1454359 T4145 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1454359 T4145 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1454360 T4145 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1454360 T4146 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@27aa81d8[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1454360 T4149 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:20833 collection:control_collection shard:shard1
   [junit4]   2> 1454361 T4115 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 1454361 T4115 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1454361 T4149 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1454364 T4149 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1454365 T4141 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1454365 T4115 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1454365 T4149 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1454365 T4149 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1454366 T4142 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C2151 name=collection1 org.apache.solr.core.SolrCore@1a3681ad url=http://127.0.0.1:20833/collection1 node=127.0.0.1:20833_ C2151_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:20833, node_name=127.0.0.1:20833_, core=collection1}
   [junit4]   2> 1454366 T4149 C:control_collection S:shard1 C2151 P20833 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:20833/collection1/
   [junit4]   2> 1454366 T4115 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1454366 T4149 C:control_collection S:shard1 C2151 P20833 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1454366 T4149 C:control_collection S:shard1 C2151 P20833 oasc.SyncStrategy.syncToMe http://127.0.0.1:20833/collection1/ has no replicas
   [junit4]   2> 1454367 T4149 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:20833/collection1/ shard1
   [junit4]   2> 1454367 T4149 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1454367 T4141 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1454367 T4153 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@ef73097 name:ZooKeeperConnection Watcher:127.0.0.1:20824/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1454367 T4115 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1454368 T4115 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1454368 T4115 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1454369 T4141 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1454370 T4142 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20833",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 1454370 T4115 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1454472 T4141 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1454472 T4153 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1454522 T4149 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:20833/collection1/ and leader is http://127.0.0.1:20833/collection1/
   [junit4]   2> 1454523 T4149 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:20833
   [junit4]   2> 1454523 T4149 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1454523 T4149 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1454523 T4149 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1454524 T4141 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1454525 T4142 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20833",
   [junit4]   2> 	  "node_name":"127.0.0.1:20833_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 3
   [junit4]   2> 1454526 T4142 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20833",
   [junit4]   2> 	  "node_name":"127.0.0.1:20833_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 1454628 T4153 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1454628 T4141 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1456038 T4115 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/cores/collection1
   [junit4]   2> 1456042 T4115 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001
   [junit4]   2> 1456043 T4115 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1456046 T4115 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:20845
   [junit4]   2> 1456046 T4115 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/cores, hostPort=20844, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/tempDir-001/jetty1, hostContext=/}
   [junit4]   2> 1456047 T4115 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 1456047 T4115 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/'
   [junit4]   2> 1456096 T4115 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/solr.xml
   [junit4]   2> 1456155 T4115 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/cores
   [junit4]   2> 1456156 T4115 oasc.CoreContainer.<init> New CoreContainer 1782224232
   [junit4]   2> 1456156 T4115 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/]
   [junit4]   2> 1456156 T4115 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/lib
   [junit4]   2> 1456157 T4115 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/lib).
   [junit4]   2> 1456171 T4115 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1456172 T4115 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1456172 T4115 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1456172 T4115 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1456173 T4115 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1456173 T4115 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1456173 T4115 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1456173 T4115 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1456174 T4115 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1456174 T4115 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1456174 T4115 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1456182 T4115 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1456183 T4115 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1456183 T4115 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1456184 T4115 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1456184 T4115 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:20824/solr
   [junit4]   2> 1456184 T4115 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1456185 T4115 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1456186 T4115 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1456187 T4165 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7e6580cf name:ZooKeeperConnection Watcher:127.0.0.1:20824 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1456188 T4115 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1456189 T4115 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1456191 T4115 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1456192 T4168 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@27f768a name:ZooKeeperConnection Watcher:127.0.0.1:20824/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1456192 T4115 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1456196 T4115 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1457200 T4115 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:20844_
   [junit4]   2> 1457200 T4115 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:20844_
   [junit4]   2> 1457203 T4115 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1457205 T4115 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/cores
   [junit4]   2> 1457207 T4115 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {loadOnStartup=true, dataDir=data/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/cores/collection1, transient=false, shard=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/cores/collection1/, schema=schema.xml, config=solrconfig.xml, coreNodeName=, collection=collection1, name=collection1}
   [junit4]   2> 1457208 T4115 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/cores/collection1/
   [junit4]   2> 1457208 T4115 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1457209 T4169 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1457209 T4169 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1457210 T4169 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1457210 T4141 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1457211 T4142 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20844",
   [junit4]   2> 	  "node_name":"127.0.0.1:20844_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1"} current state version: 4
   [junit4]   2> 1457211 T4142 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20844",
   [junit4]   2> 	  "node_name":"127.0.0.1:20844_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1"}
   [junit4]   2> 1457211 T4142 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 1457211 T4142 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1457212 T4141 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1457212 T4168 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1457212 T4153 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1458210 T4169 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1458211 T4169 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1458211 T4169 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1458211 T4169 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1458212 T4169 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1458212 T4169 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/cores/collection1/'
   [junit4]   2> 1458241 T4169 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1458244 T4169 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1458255 T4169 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1458274 T4169 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1458314 T4169 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1458415 T4169 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1458423 T4169 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1458847 T4169 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1458860 T4169 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1458862 T4169 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1458869 T4169 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1458873 T4169 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1458876 T4169 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1458876 T4169 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1458877 T4169 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1458878 T4169 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1458878 T4169 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1458878 T4169 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1458878 T4169 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1458879 T4169 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1458879 T4169 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 1458879 T4169 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5fa863b5
   [junit4]   2> 1458880 T4169 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/cores/collection1/data
   [junit4]   2> 1458880 T4169 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 1458880 T4169 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1458880 T4169 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 1458881 T4169 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=10, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.15179102665711916]
   [junit4]   2> 1458881 T4169 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@76847976 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73426ec3),segFN=segments_1,generation=1}
   [junit4]   2> 1458881 T4169 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1458886 T4169 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1458886 T4169 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1458887 T4169 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1458887 T4169 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1458887 T4169 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1458887 T4169 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1458888 T4169 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1458888 T4169 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1458888 T4169 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1458902 T4169 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1458904 T4169 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1458905 T4169 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1458907 T4169 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1458912 T4169 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/system,/admin/segments,/admin/file,/admin/mbeans,standard,/admin/properties,/admin/threads,/update/json/docs,/admin/luke,/get,/config,/update/csv,/update/json,/admin/ping,/update,/schema,/replication,/admin/logging,/admin/plugins
   [junit4]   2> 1458914 T4169 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1458915 T4169 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1458915 T4169 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1458916 T4169 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1458916 T4169 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1458917 T4169 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=50, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5958199606090643]
   [junit4]   2> 1458918 T4169 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@76847976 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73426ec3),segFN=segments_1,generation=1}
   [junit4]   2> 1458918 T4169 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1458918 T4169 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@1d87e561[collection1] main
   [junit4]   2> 1458918 T4169 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1458919 T4169 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1458919 T4169 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1458919 T4169 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1458920 T4169 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1458920 T4169 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1458920 T4169 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1458920 T4169 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1458920 T4169 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1458921 T4169 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1458922 T4170 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1d87e561[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1458922 T4169 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1458923 T4173 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:20844 collection:collection1 shard:shard1
   [junit4]   2> 1458923 T4115 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 1458923 T4115 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1458924 T4173 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1458927 T4173 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1458928 T4141 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1458928 T4173 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1458929 T4173 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C2152 name=collection1 org.apache.solr.core.SolrCore@4d26573b url=http://127.0.0.1:20844/collection1 node=127.0.0.1:20844_ C2152_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:20844, node_name=127.0.0.1:20844_, core=collection1}
   [junit4]   2> 1458929 T4173 C:collection1 S:shard1 C2152 P20844 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:20844/collection1/
   [junit4]   2> 1458929 T4142 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 1458929 T4173 C:collection1 S:shard1 C2152 P20844 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1458930 T4173 C:collection1 S:shard1 C2152 P20844 oasc.SyncStrategy.syncToMe http://127.0.0.1:20844/collection1/ has no replicas
   [junit4]   2> 1458930 T4173 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:20844/collection1/ shard1
   [junit4]   2> 1458930 T4173 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1458930 T4168 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1458930 T4153 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1458930 T4141 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1458933 T4141 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1458934 T4142 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20844",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 1459036 T4141 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1459036 T4153 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1459036 T4168 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1459086 T4173 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:20844/collection1/ and leader is http://127.0.0.1:20844/collection1/
   [junit4]   2> 1459087 T4173 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:20844
   [junit4]   2> 1459087 T4173 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1459087 T4173 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1459087 T4173 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1459088 T4141 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1459089 T4142 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20844",
   [junit4]   2> 	  "node_name":"127.0.0.1:20844_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1"} current state version: 7
   [junit4]   2> 1459089 T4142 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20844",
   [ju

[...truncated too long message...]

]   2> 1611043 T6476 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:21210 21210
   [junit4]   2> 1611246 T6477 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:21210 21210
   [junit4]   2> 1611247 T6477 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1611247 T6476 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:21221/
   [junit4]   2> 1611248 T6476 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:21213/
   [junit4]   2> 1611248 T6476 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:21229/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=691015E6B7971E47 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=nl_NL -Dtests.timezone=America/St_Lucia -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   67.5s J2 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:21221/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([691015E6B7971E47:E1442A3C196B73BF]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:21221/collection1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:21221 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 54 more
   [junit4]   2> 1611287 T6476 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 691015E6B7971E47-001
   [junit4]   2> NOTE: test params are: codec=FastDecompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=4775, maxDocsPerChunk=4, blockSize=57), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=4775, blockSize=57)), sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=nl_NL, timezone=America/St_Lucia
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=183747848,total=338165760
   [junit4]   2> NOTE: All tests run in this JVM: [DistributedIntervalFacetingTest, SpellCheckCollatorTest, MultiTermTest, PreAnalyzedFieldTest, SpatialHeatmapFacetsTest, ConnectionManagerTest, RAMDirectoryFactoryTest, TestSolrXml, TestRecovery, TestWriterPerf, TestSolrDeletionPolicy1, SOLR749Test, TestConfigOverlay, TestDynamicFieldCollectionResource, TestSearchPerf, DefaultValueUpdateProcessorTest, TestCoreContainer, UUIDFieldTest, DeleteReplicaTest, BJQParserTest, TestSolrIndexConfig, TestFieldResource, ClusterStateUpdateTest, TestSort, SuggesterTSTTest, TestClusterStateMutator, SolrCoreTest, TestJoin, CachingDirectoryFactoryTest, UniqFieldsUpdateProcessorFactoryTest, SolrIndexSplitterTest, TestSearcherReuse, SimpleFacetsTest, AtomicUpdatesTest, TestDistributedSearch, FieldAnalysisRequestHandlerTest, SolrPluginUtilsTest, SSLMigrationTest, TestDFRSimilarityFactory, TestLRUCache, RankQueryTest, AddBlockUpdateTest, DocValuesMissingTest, XmlUpdateRequestHandlerTest, RollingRestartTest, TestCodecSupport, QueryResultKeyTest, SharedFSAutoReplicaFailoverTest, TestExpandComponent, TestIntervalFaceting, DistributedMLTComponentTest, TestConfigSets, FieldMutatingUpdateProcessorTest, CoreAdminRequestStatusTest, DistributedDebugComponentTest, DistributedSpellCheckComponentTest, TestComponentsName, TestRebalanceLeaders, LeaderFailoverAfterPartitionTest, BasicDistributedZkTest, ChaosMonkeyNothingIsSafeTest, TestSolrConfigHandlerCloud, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J2 in 67.50s, 1 test, 1 error <<< FAILURES!

[...truncated 690 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 53 minutes 25 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 218077853 bytes
Compression is 0.0%
Took 38 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2883 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2883/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:49400/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:49400/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([613D1676A5A50840:E96929AC0B5965B8]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([613D1676A5A50840:E96929AC0B5965B8]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([613D1676A5A50840:E96929AC0B5965B8]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9361 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/init-core-data-001
   [junit4]   2> 410808 T986 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /b/tv
   [junit4]   2> 410816 T986 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 410817 T987 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 410917 T986 oasc.ZkTestServer.run start zk server on port:11040
   [junit4]   2> 410918 T986 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 410919 T986 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 410923 T994 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4b310ac6 name:ZooKeeperConnection Watcher:127.0.0.1:11040 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 410924 T986 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 410924 T986 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 410925 T986 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 410929 T986 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 410930 T986 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 410931 T997 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4cca2544 name:ZooKeeperConnection Watcher:127.0.0.1:11040/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 410932 T986 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 410932 T986 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 410933 T986 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 410935 T986 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 410937 T986 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 410938 T986 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 410940 T986 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 410940 T986 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 410943 T986 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 410944 T986 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 410946 T986 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 410947 T986 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 410949 T986 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 410949 T986 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 410951 T986 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 410951 T986 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 410953 T986 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 410953 T986 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 410955 T986 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 410956 T986 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 410957 T986 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 410958 T986 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 410960 T986 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 410960 T986 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 410962 T986 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 410963 T986 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 410964 T986 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 410965 T986 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 412310 T986 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/cores/collection1
   [junit4]   2> 412315 T986 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 412320 T986 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:11079
   [junit4]   2> 412320 T986 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/b/tv, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/tempDir-001/control/data, hostPort=11078, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/cores}
   [junit4]   2> 412321 T986 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3b061299
   [junit4]   2> 412321 T986 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/'
   [junit4]   2> 412360 T986 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/solr.xml
   [junit4]   2> 412406 T986 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/cores
   [junit4]   2> 412407 T986 oasc.CoreContainer.<init> New CoreContainer 1556104529
   [junit4]   2> 412407 T986 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/]
   [junit4]   2> 412407 T986 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/lib
   [junit4]   2> 412408 T986 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/lib).
   [junit4]   2> 412421 T986 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 412421 T986 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 412421 T986 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 412422 T986 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 412422 T986 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 412422 T986 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 412422 T986 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 412423 T986 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 412423 T986 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 412423 T986 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 412424 T986 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 412424 T986 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 412425 T986 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 412425 T986 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 412425 T986 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 412425 T986 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:11040/solr
   [junit4]   2> 412426 T986 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 412426 T986 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 412427 T986 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 412429 T1009 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4eb0ad7c name:ZooKeeperConnection Watcher:127.0.0.1:11040 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 412430 T986 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 412430 T986 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 412432 T986 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 412434 T1012 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5c731511 name:ZooKeeperConnection Watcher:127.0.0.1:11040/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 412434 T986 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 412435 T986 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 412437 T986 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 412439 T986 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 412441 T986 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 412443 T986 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 412445 T986 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 412446 T986 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 412447 T986 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 412448 T986 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:11078_b%2Ftv
   [junit4]   2> 412449 T986 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:11078_b%2Ftv
   [junit4]   2> 412450 T986 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 412452 T986 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 412453 T986 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 412454 T986 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:11078_b%2Ftv
   [junit4]   2> 412455 T986 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 412456 T986 oasc.Overseer.start Overseer (id=93589584465756163-127.0.0.1:11078_b%2Ftv-n_0000000000) starting
   [junit4]   2> 412458 T986 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 412463 T986 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 412464 T986 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 412464 T1013 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 412464 T1014 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 412469 T986 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/cores
   [junit4]   2> 412471 T986 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, collection=control_collection, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/cores/collection1, dataDir=data/, coreNodeName=, transient=false, loadOnStartup=true, schema=schema.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/cores/collection1/, name=collection1, config=solrconfig.xml}
   [junit4]   2> 412472 T986 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/cores/collection1/
   [junit4]   2> 412472 T986 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 412474 T1016 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 412474 T1016 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 412475 T1016 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 412475 T1012 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 412476 T1013 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:11078_b%2Ftv",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11078/b/tv"} current state version: 0
   [junit4]   2> 412477 T1013 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:11078_b%2Ftv",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11078/b/tv"}
   [junit4]   2> 412477 T1013 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 412477 T1013 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 412479 T1012 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 413475 T1016 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 413476 T1016 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 413476 T1016 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 413476 T1016 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 413477 T1016 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 413477 T1016 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/cores/collection1/'
   [junit4]   2> 413490 T1016 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 413492 T1016 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 413501 T1016 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 413517 T1016 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 413552 T1016 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 413553 T1016 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 413561 T1016 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 413848 T1016 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 413860 T1016 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 413862 T1016 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 413871 T1016 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 413875 T1016 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 413878 T1016 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 413878 T1016 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 413878 T1016 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 413879 T1016 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 413879 T1016 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 413879 T1016 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 413880 T1016 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 413880 T1016 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 413880 T1016 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/cores/collection1/data/
   [junit4]   2> 413880 T1016 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@592b94df
   [junit4]   2> 413881 T1016 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/cores/collection1/data
   [junit4]   2> 413881 T1016 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/cores/collection1/data/index/
   [junit4]   2> 413881 T1016 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 413882 T1016 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/cores/collection1/data/index
   [junit4]   2> 413882 T1016 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=32, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 413883 T1016 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@dd6ce4 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4917ed2d),segFN=segments_1,generation=1}
   [junit4]   2> 413883 T1016 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 413888 T1016 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 413888 T1016 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 413888 T1016 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 413889 T1016 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 413889 T1016 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 413889 T1016 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 413889 T1016 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 413890 T1016 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 413890 T1016 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 413902 T1016 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 413904 T1016 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 413906 T1016 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 413907 T1016 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 413912 T1016 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/luke,/schema,/config,/update,/update/csv,/update/json,/replication,standard,/admin/segments,/admin/ping,/admin/plugins,/admin/file,/update/json/docs,/admin/logging,/admin/mbeans,/admin/threads,/admin/system,/admin/properties,/get
   [junit4]   2> 413913 T1016 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 413914 T1016 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 413915 T1016 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 413916 T1016 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 413916 T1016 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 413916 T1016 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=33, maxMergeAtOnceExplicit=13, maxMergedSegmentMB=17.318359375, floorSegmentMB=0.9638671875, forceMergeDeletesPctAllowed=8.785740902982491, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3474586037465189
   [junit4]   2> 413917 T1016 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@dd6ce4 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4917ed2d),segFN=segments_1,generation=1}
   [junit4]   2> 413917 T1016 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 413917 T1016 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@733b8bc1[collection1] main
   [junit4]   2> 413918 T1016 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 413918 T1016 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 413918 T1016 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 413919 T1016 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 413919 T1016 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 413919 T1016 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 413919 T1016 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 413920 T1016 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 413920 T1016 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 413920 T1016 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 413921 T1017 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@733b8bc1[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 413921 T1016 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 413922 T1020 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:11078/b/tv collection:control_collection shard:shard1
   [junit4]   2> 413922 T986 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 413923 T986 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 413923 T1020 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 413925 T986 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 413926 T986 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 413927 T1020 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 413927 T1024 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5fc5dbc7 name:ZooKeeperConnection Watcher:127.0.0.1:11040/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 413928 T986 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 413928 T1012 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 413928 T1020 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 413928 T986 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 413928 T1020 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 413928 T1013 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 413928 T986 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C257 name=collection1 org.apache.solr.core.SolrCore@a455eab url=http://127.0.0.1:11078/b/tv/collection1 node=127.0.0.1:11078_b%2Ftv C257_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:11078_b%2Ftv, state=down, core=collection1, base_url=http://127.0.0.1:11078/b/tv}
   [junit4]   2> 413929 T1020 C:control_collection S:shard1 C257 P11078 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:11078/b/tv/collection1/
   [junit4]   2> 413929 T1020 C:control_collection S:shard1 C257 P11078 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 413929 T1020 C:control_collection S:shard1 C257 P11078 oasc.SyncStrategy.syncToMe http://127.0.0.1:11078/b/tv/collection1/ has no replicas
   [junit4]   2> 413929 T1020 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:11078/b/tv/collection1/ shard1
   [junit4]   2> 413930 T1020 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 413931 T986 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 413932 T1012 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 413933 T1013 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11078/b/tv",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 1
   [junit4]   2> 414035 T1024 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 414035 T1012 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 414085 T1020 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:11078/b/tv/collection1/ and leader is http://127.0.0.1:11078/b/tv/collection1/
   [junit4]   2> 414086 T1020 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:11078/b/tv
   [junit4]   2> 414086 T1020 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 414086 T1020 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 414086 T1020 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 414087 T1012 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 414088 T1013 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:11078_b%2Ftv",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11078/b/tv"} current state version: 2
   [junit4]   2> 414088 T1013 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:11078_b%2Ftv",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11078/b/tv"}
   [junit4]   2> 414191 T1012 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 414191 T1024 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 414895 T986 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/cores/collection1
   [junit4]   2> 414897 T986 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001
   [junit4]   2> 414898 T986 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 414901 T986 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:11085
   [junit4]   2> 414901 T986 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/cores, hostContext=/b/tv, hostPort=11084, solrconfig=solrconfig.xml}
   [junit4]   2> 414902 T986 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3b061299
   [junit4]   2> 414902 T986 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/'
   [junit4]   2> 414935 T986 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/solr.xml
   [junit4]   2> 414972 T986 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/cores
   [junit4]   2> 414973 T986 oasc.CoreContainer.<init> New CoreContainer 2063040257
   [junit4]   2> 414973 T986 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/]
   [junit4]   2> 414973 T986 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/lib
   [junit4]   2> 414974 T986 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/lib).
   [junit4]   2> 414984 T986 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 414984 T986 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 414984 T986 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 414985 T986 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 414985 T986 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 414985 T986 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 414985 T986 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 414985 T986 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 414986 T986 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 414986 T986 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 414986 T986 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 414987 T986 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 414987 T986 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 414987 T986 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 414987 T986 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 414988 T986 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:11040/solr
   [junit4]   2> 414988 T986 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 414988 T986 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 414989 T986 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 414990 T1036 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@73f56cc0 name:ZooKeeperConnection Watcher:127.0.0.1:11040 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 414991 T986 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 414992 T986 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 414994 T986 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 414996 T1039 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@11cd7ee0 name:ZooKeeperConnection Watcher:127.0.0.1:11040/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 414996 T986 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 414999 T986 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 416003 T986 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:11084_b%2Ftv
   [junit4]   2> 416004 T986 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:11084_b%2Ftv
   [junit4]   2> 416007 T986 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 416009 T986 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/cores
   [junit4]   2> 416011 T986 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, transient=false, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/cores/collection1, config=solrconfig.xml, collection=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/cores/collection1/, dataDir=data/, coreNodeName=, shard=, loadOnStartup=true}
   [junit4]   2> 416011 T986 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/cores/collection1/
   [junit4]   2> 416012 T986 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 416013 T1040 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 416014 T1040 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 416014 T1012 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 416014 T1040 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 416015 T1013 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:11084_b%2Ftv",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11084/b/tv"} current state version: 3
   [junit4]   2> 416016 T1013 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:11084_b%2Ftv",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11084/b/tv"}
   [junit4]   2> 416016 T1013 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 416016 T1013 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 416017 T1039 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 416017 T1024 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 416017 T1012 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 417015 T1040 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 417016 T1040 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 417016 T1040 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 417016 T1040 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 417017 T1040 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 417017 T1040 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/cores/collection1/'
   [junit4]   2> 417030 T1040 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 417032 T1040 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 417041 T1040 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 417056 T1040 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 417091 T1040 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 417092 T1040 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 417100 T1040 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 417378 T1040 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 417397 T1040 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 417399 T1040 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 417408 T1040 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 417412 T1040 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 417415 T1040 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 417416 T1040 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 417416 T1040 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 417416 T1040 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 417417 T1040 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 417417 T1040 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 417417 T1040 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 417417 T1040 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 417418 T1040 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 417418 T1040 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@592b94df
   [junit4]   2> 417419 T1040 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/cores/collection1/data
   [junit4]   2> 417419 T1040 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 417419 T1040 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 417419 T1040 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 417420 T1040 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=32, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 417420 T1040 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6ae174eb lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@77c3271e),segFN=segments_1,generation=1}
   [junit4]   2> 417420 T1040 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 417426 T1040 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 417427 T1040 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 417427 T1040 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 417427 T1040 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 417427 T1040 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 417427 T1040 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 417428 T1040 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 417428 T1040 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 417428 T1040 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 417441 T1040 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 417443 T1040 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 417444 T1040 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 417446 T1040 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 417452 T1040 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/luke,/schema,/config,/update,/update/csv,/update/json,/replication,standard,/admin/segments,/admin/ping,/admin/plugins,/admin/file,/update/json/docs,/admin/logging,/admin/mbeans,/admin/threads,/admin/system,/admin/properties,/get
   [junit4]   2> 417454 T1040 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 417455 T1040 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 417455 T1040 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 417456 T1040 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 417456 T1040 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 417457 T1040 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=33, maxMergeAtOnceExplicit=13, maxMergedSegmentMB=17.318359375, floorSegmentMB=0.9638671875, forceMergeDeletesPctAllowed=8.785740902982491, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3474586037465189
   [junit4]   2> 417458 T1040 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6ae174eb lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@77c3271e),segFN=segments_1,generation=1}
   [junit4]   2> 417458 T1040 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 417458 T1040 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@32f78197[collection1] main
   [junit4]   2> 417458 T1040 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 417459 T1040 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 417459 T1040 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 417459 T1040 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 417460 T1040 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 417460 T1040 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 417460 T1040 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 417460 T1040 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 417461 T1040 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 417461 T1040 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 417462 T1041 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@32f78197[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 417462 T1040 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 417463 T1044 C:collection1 S:shard2 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:11084/b/tv collection:collection1 shard:shard2
   [junit4]   2> 417463 T986 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 417464 T986 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 417464 T1044 C:collection1 S:shard2 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 417467 T1044 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 417469 T1012 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 417469 T1044 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 417469 T1044 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C258 name=collection1 org.apache.solr.core.SolrCore@61b40f8c url=http://127.0.0.1:11084/b/tv/collection1 node=127.0.0.1:11084_b%2Ftv C258_STATE=coll:collection1 core:collection1 props:{node_name=127.0.0.1:11084_b%2Ftv, state=down, core=collection1, base_url=http://127.0.0.1:11084/b/tv}
   [junit4]   2> 417469 T1044 C:collection1 S:shard2 C258 P11084 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:11084/b/tv/collection1/
   [junit4]   2> 417469 T1013 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1"} current state version: 4
   [junit4]   2> 417469 T1044 C:collection1 S:shard2 C258 P11084 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 417470 T1044 C:collection1 S:shard2 C258 P11084 oasc.SyncStrategy.syncToMe http://127.0.0.1:11084/b/tv/collection1/ has no replicas
   [junit4]   2> 417470 T1044 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:11084/b/tv/collection1/ shard2
   [junit4]   2> 417470 T1044 C:collection1 S:shard2 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 417473 T1012 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 417473 T1013 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11084/b/tv",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 4
   [junit4]   2> 417576 T1012 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 417576 T1039 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 417576 T1024 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 417625 T1044 C:collection1 S:shard2 oasc.ZkController.register We are http://127.0.0.1:11084/b/tv/collection1/ and leader is http://127.0.0.1:11084/b/tv/collection1/
   [junit4]   2> 417626 T1044 C:collection1 S:shard2 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:11084/b/tv
   [junit4]   2> 417626 T1044 C:collection1 S:shard2 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 417626 T1044 C:collection1 S:shard2 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 417626 T1044 C:collection1 S:shard2 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 417627 T1012 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 417628 T1013 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:11084_b%2Ftv",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11084/b/tv"} current state version: 5
   [junit4]   2> 417628 T1013 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:11084_b%2Ftv",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11084/b/tv"}
   [junit4]   2> 417731 T1039 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 417731 T1024 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 417731 T1012 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 418404 T986 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-2-001/cores/collection1
   [junit4]   2> 418406 T986 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-2-001
   [junit4]   2> 418407 T986 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 418409 T986 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:11089
   [junit4]   2> 418409 T986 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-2-001/cores, hostContext=/b/tv, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostPort=11088}
   [junit4]   2> 418410 T986 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3b061299
   [junit4]   2> 418410 T986 oasc.SolrResourceLoader

[...truncated too long message...]

red, but Solr cannot talk to ZK
   [junit4]   2> 928792 T5179 C:control_collection S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 928955 T5179 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:32175 32175
   [junit4]   2> 929019 T5180 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:32175 32175
   [junit4]   2> 929021 T5180 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 929021 T5179 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:32179/
   [junit4]   2> 929021 T5179 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:32198/
   [junit4]   2> 929022 T5179 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 6 connections to: http://127.0.0.1:32188/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=613D1676A5A50840 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ar_LB -Dtests.timezone=Indian/Cocos -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   75.9s J1 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([613D1676A5A50840:E96929AC0B5965B8]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 929028 T5179 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 613D1676A5A50840-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {id=Lucene50(blocksize=128), a_t=PostingsFormat(name=Memory doPackFST= false), timestamp=PostingsFormat(name=Memory doPackFST= false), intDefault=PostingsFormat(name=Memory doPackFST= false), text=PostingsFormat(name=Direct), range_facet_l=PostingsFormat(name=Memory doPackFST= false), multiDefault=Lucene50(blocksize=128), _version_=PostingsFormat(name=Memory doPackFST= false)}, docValues:{timestamp=DocValuesFormat(name=Asserting)}, sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=ar_LB, timezone=Indian/Cocos
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=129014776,total=289406976
   [junit4]   2> NOTE: All tests run in this JVM: [TestSchemaNameResource, TestRTGBase, TestEmbeddedSolrServerConstructors, MultiTermTest, AssignTest, OverriddenZkACLAndCredentialsProvidersTest, TestSolrQueryParserDefaultOperatorResource, DistribDocExpirationUpdateProcessorTest, DistributedTermsComponentTest, DocValuesTest, HighlighterTest, ShardSplitTest, TestSort, ReplicationFactorTest, TestConfigOverlay, CloudExitableDirectoryReaderTest, TestCollationField, NumericFieldsTest, TestDynamicFieldCollectionResource, DistanceUnitsTest, TimeZoneUtilsTest, FieldMutatingUpdateProcessorTest, DirectSolrSpellCheckerTest, TestReplicaProperties, TestPhraseSuggestions, UniqFieldsUpdateProcessorFactoryTest, TestPerFieldSimilarity, TestSolrConfigHandlerCloud, SolrCoreTest, AnalyticsMergeStrategyTest, TestSolr4Spatial2, SolrCmdDistributorTest, TestHdfsUpdateLog, AnalysisErrorHandlingTest, DistributedQueryComponentCustomSortTest, TestRecovery, DistributedSpellCheckComponentTest, TestLFUCache, FastVectorHighlighterTest, BadComponentTest, TestSchemaVersionResource, SpellCheckCollatorTest, TestCursorMarkWithoutUniqueKey, TestReplicationHandlerBackup, TestDistributedGrouping, OutOfBoxZkACLAndCredentialsProvidersTest, TestLRUCache, TestSimpleQParserPlugin, AddSchemaFieldsUpdateProcessorFactoryTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J1 in 75.92s, 1 test, 1 error <<< FAILURES!

[...truncated 937 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 54 minutes 32 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 219492621 bytes
Compression is 0.0%
Took 42 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2882 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2882/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:16230/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16230/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([46171878E3F1D9B8:CE4327A24D0DB440]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:23333/i_zk/cc/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:23333/i_zk/cc/collection1
	at __randomizedtesting.SeedInfo.seed([46171878E3F1D9B8:CE4327A24D0DB440]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:23333/i_zk/cc/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:56796/zk/g/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:56796/zk/g/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([46171878E3F1D9B8:CE4327A24D0DB440]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:56796/zk/g/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9405 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/init-core-data-001
   [junit4]   2> 119902 T346 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /i_zk/cc
   [junit4]   2> 119919 T346 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 119920 T347 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 120021 T346 oasc.ZkTestServer.run start zk server on port:23310
   [junit4]   2> 120022 T346 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 120023 T346 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 120028 T354 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7501dca0 name:ZooKeeperConnection Watcher:127.0.0.1:23310 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 120029 T346 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 120029 T346 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 120030 T346 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 120034 T346 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 120035 T346 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 120037 T357 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@643ff3b2 name:ZooKeeperConnection Watcher:127.0.0.1:23310/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 120037 T346 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 120038 T346 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 120038 T346 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 120042 T346 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 120046 T346 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 120048 T346 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 120052 T346 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 120053 T346 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 120058 T346 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 120059 T346 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 120063 T346 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 120064 T346 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 120067 T346 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 120068 T346 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 120072 T346 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 120073 T346 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 120076 T346 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 120077 T346 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 120080 T346 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 120081 T346 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 120084 T346 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 120085 T346 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 120088 T346 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 120089 T346 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 120093 T346 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 120094 T346 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 120097 T346 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 120098 T346 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 121832 T346 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/cores/collection1
   [junit4]   2> 121837 T346 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 121841 T346 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:23317
   [junit4]   2> 121841 T346 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/cores, hostContext=/i_zk/cc, hostPort=23316}
   [junit4]   2> 121842 T346 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 121842 T346 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/'
   [junit4]   2> 121885 T346 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/solr.xml
   [junit4]   2> 121935 T346 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/cores
   [junit4]   2> 121936 T346 oasc.CoreContainer.<init> New CoreContainer 1844476828
   [junit4]   2> 121936 T346 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/]
   [junit4]   2> 121937 T346 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/lib
   [junit4]   2> 121937 T346 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/lib).
   [junit4]   2> 121948 T346 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 121948 T346 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 121948 T346 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 121949 T346 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 121949 T346 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 121949 T346 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 121949 T346 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 121950 T346 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 121950 T346 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 121950 T346 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 121950 T346 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 121951 T346 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 121951 T346 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 121951 T346 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 121952 T346 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 121952 T346 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:23310/solr
   [junit4]   2> 121952 T346 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 121952 T346 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 121953 T346 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 121955 T369 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@262a33a1 name:ZooKeeperConnection Watcher:127.0.0.1:23310 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 121956 T346 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 121957 T346 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 121960 T346 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 121961 T372 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2f756f1 name:ZooKeeperConnection Watcher:127.0.0.1:23310/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 121961 T346 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 121963 T346 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 121966 T346 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 121969 T346 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 121972 T346 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 121975 T346 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 121978 T346 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 121980 T346 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 121982 T346 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 121983 T346 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:23316_i_zk%2Fcc
   [junit4]   2> 121984 T346 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:23316_i_zk%2Fcc
   [junit4]   2> 121987 T346 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 121989 T346 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 121991 T346 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 121993 T346 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:23316_i_zk%2Fcc
   [junit4]   2> 121993 T346 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 121995 T346 oasc.Overseer.start Overseer (id=93588620005015555-127.0.0.1:23316_i_zk%2Fcc-n_0000000000) starting
   [junit4]   2> 121998 T346 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 122007 T346 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 122008 T374 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 122008 T346 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 122009 T373 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 122014 T346 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/cores
   [junit4]   2> 122030 T346 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, loadOnStartup=true, collection=control_collection, coreNodeName=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/cores/collection1/, dataDir=data/, shard=, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/cores/collection1, schema=schema.xml, transient=false}
   [junit4]   2> 122031 T346 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/cores/collection1/
   [junit4]   2> 122031 T346 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 122033 T376 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 122033 T376 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 122043 T372 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 122043 T376 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 122051 T373 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:23316_i_zk%2Fcc",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23316/i_zk/cc"} current state version: 0
   [junit4]   2> 122099 T373 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:23316_i_zk%2Fcc",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23316/i_zk/cc"}
   [junit4]   2> 122107 T373 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 122140 T373 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 122160 T372 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 123045 T376 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 123045 T376 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 123046 T376 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 123046 T376 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 123047 T376 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 123048 T376 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/cores/collection1/'
   [junit4]   2> 123097 T376 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 123115 T376 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 123145 T376 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 123165 T376 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 123202 T376 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 123204 T376 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 123214 T376 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 123622 T376 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 123643 T376 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 123646 T376 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 123664 T376 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 123670 T376 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 123675 T376 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 123676 T376 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 123676 T376 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 123677 T376 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 123678 T376 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 123678 T376 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 123678 T376 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 123679 T376 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 123679 T376 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/cores/collection1/data/
   [junit4]   2> 123680 T376 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3f430afc
   [junit4]   2> 123681 T376 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/cores/collection1/data
   [junit4]   2> 123681 T376 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/cores/collection1/data/index/
   [junit4]   2> 123682 T376 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 123682 T376 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/cores/collection1/data/index
   [junit4]   2> 123683 T376 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=32, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.45963115368360286]
   [junit4]   2> 123684 T376 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@10b5b606 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@9c14c7c),segFN=segments_1,generation=1}
   [junit4]   2> 123684 T376 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 123689 T376 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 123777 T376 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 123778 T376 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 123778 T376 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 123778 T376 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 123778 T376 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 123779 T376 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 123779 T376 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 123779 T376 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 123806 T376 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 123809 T376 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 123811 T376 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 123813 T376 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 123818 T376 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/plugins,/admin/segments,/admin/mbeans,/admin/ping,/update/csv,/update/json,/get,/config,/admin/threads,standard,/replication,/admin/logging,/admin/properties,/update,/update/json/docs,/admin/file,/admin/luke,/admin/system,/schema
   [junit4]   2> 123819 T376 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 123821 T376 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 123821 T376 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 123823 T376 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 123823 T376 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 123824 T376 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=15, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6488594503462402]
   [junit4]   2> 123824 T376 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@10b5b606 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@9c14c7c),segFN=segments_1,generation=1}
   [junit4]   2> 123824 T376 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 123825 T376 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@36b83e64[collection1] main
   [junit4]   2> 123825 T376 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 123826 T376 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 123827 T376 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 123840 T376 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 123841 T376 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 123841 T376 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 123842 T376 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 123842 T376 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 123842 T376 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 123843 T376 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 123844 T377 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@36b83e64[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 123851 T376 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 123852 T380 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:23316/i_zk/cc collection:control_collection shard:shard1
   [junit4]   2> 123853 T346 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 123853 T346 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 123867 T346 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 123868 T346 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 123870 T384 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4e606c8b name:ZooKeeperConnection Watcher:127.0.0.1:23310/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 123871 T346 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 123871 T346 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 123871 T346 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 123878 T346 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 123879 T346 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 123879 T346 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 123880 T346 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 123882 T387 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4f02246c name:ZooKeeperConnection Watcher:127.0.0.1:23310/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 123882 T346 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 123882 T346 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 123885 T372 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 123886 T373 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 1
   [junit4]   2> 123887 T373 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 123889 T373 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 123891 T372 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 123891 T384 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 123903 T380 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 123911 T380 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 123914 T372 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 123914 T380 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 123915 T380 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 123915 T373 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 2
   [junit4]   2> ASYNC  NEW_CORE C38 name=collection1 org.apache.solr.core.SolrCore@5c0cc960 url=http://127.0.0.1:23316/i_zk/cc/collection1 node=127.0.0.1:23316_i_zk%2Fcc C38_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:23316_i_zk%2Fcc, state=down, core=collection1, base_url=http://127.0.0.1:23316/i_zk/cc}
   [junit4]   2> 123925 T380 C:control_collection S:shard1 C38 P23316 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:23316/i_zk/cc/collection1/
   [junit4]   2> 123926 T380 C:control_collection S:shard1 C38 P23316 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 123926 T380 C:control_collection S:shard1 C38 P23316 oasc.SyncStrategy.syncToMe http://127.0.0.1:23316/i_zk/cc/collection1/ has no replicas
   [junit4]   2> 123927 T380 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:23316/i_zk/cc/collection1/ shard1
   [junit4]   2> 123941 T380 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 123947 T372 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 123948 T373 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23316/i_zk/cc",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 124054 T384 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 124054 T372 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 124101 T380 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:23316/i_zk/cc/collection1/ and leader is http://127.0.0.1:23316/i_zk/cc/collection1/
   [junit4]   2> 124101 T380 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:23316/i_zk/cc
   [junit4]   2> 124101 T380 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 124102 T380 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 124102 T380 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 124104 T372 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 124105 T373 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:23316_i_zk%2Fcc",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23316/i_zk/cc"} current state version: 3
   [junit4]   2> 124106 T373 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:23316_i_zk%2Fcc",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23316/i_zk/cc"}
   [junit4]   2> 124212 T372 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 124212 T384 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 126006 T346 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/cores/collection1
   [junit4]   2> 126011 T346 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001
   [junit4]   2> 126012 T346 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 126014 T346 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:23334
   [junit4]   2> 126015 T346 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/cores, hostPort=23333, solrconfig=solrconfig.xml, hostContext=/i_zk/cc, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/tempDir-001/jetty1}
   [junit4]   2> 126016 T346 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 126016 T346 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/'
   [junit4]   2> 126082 T346 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/solr.xml
   [junit4]   2> 126165 T346 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/cores
   [junit4]   2> 126166 T346 oasc.CoreContainer.<init> New CoreContainer 294585126
   [junit4]   2> 126167 T346 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/]
   [junit4]   2> 126167 T346 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/lib
   [junit4]   2> 126168 T346 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/lib).
   [junit4]   2> 126182 T346 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 126183 T346 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 126183 T346 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 126184 T346 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 126184 T346 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 126184 T346 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 126185 T346 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 126185 T346 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 126186 T346 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 126186 T346 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 126186 T346 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 126187 T346 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 126187 T346 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 126188 T346 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 126188 T346 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 126189 T346 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:23310/solr
   [junit4]   2> 126189 T346 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 126189 T346 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 126190 T346 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 126192 T399 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@78e4cf1a name:ZooKeeperConnection Watcher:127.0.0.1:23310 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 126193 T346 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 126194 T346 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 126196 T346 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 126198 T402 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@22a30e91 name:ZooKeeperConnection Watcher:127.0.0.1:23310/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 126198 T346 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 126204 T346 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 127212 T346 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:23333_i_zk%2Fcc
   [junit4]   2> 127213 T346 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:23333_i_zk%2Fcc
   [junit4]   2> 127216 T346 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 127220 T346 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/cores
   [junit4]   2> 127222 T346 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/cores/collection1/, dataDir=data/, transient=false, config=solrconfig.xml, shard=, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/cores/collection1, coreNodeName=, name=collection1, loadOnStartup=true}
   [junit4]   2> 127223 T346 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/cores/collection1/
   [junit4]   2> 127223 T346 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 127226 T403 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 127226 T403 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 127228 T372 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 127229 T403 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 127229 T403 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 127229 T373 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:23333_i_zk%2Fcc",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23333/i_zk/cc"} current state version: 4
   [junit4]   2> 127230 T403 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 127230 T373 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:23333_i_zk%2Fcc",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23333/i_zk/cc"}
   [junit4]   2> 127230 T373 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 127231 T373 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 127233 T373 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 127237 T403 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 127237 T403 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 127238 T403 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 127238 T403 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 127238 T403 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 127238 T403 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 127239 T403 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 127239 T403 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/cores/collection1/'
   [junit4]   2> 127252 T403 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 127254 T403 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 127263 T403 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 127280 T403 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 127318 T403 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 127319 T403 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 127329 T403 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 127637 T403 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 127650 T403 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 127652 T403 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 127665 T403 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 127669 T403 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 127672 T403 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 127673 T403 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 127673 T403 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 127673 T403 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 127674 T403 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 127674 T403 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 127675 T403 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 127675 T403 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 127675 T403 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 127675 T403 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3f430afc
   [junit4]   2> 127676 T403 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/cores/collection1/data
   [junit4]   2> 127676 T403 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 127677 T403 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 127677 T403 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 127677 T403 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=32, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.45963115368360286]
   [junit4]   2> 127678 T403 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1dd9940f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@37877418),segFN=segments_1,generation=1}
   [junit4]   2> 127678 T403 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 127685 T403 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 127685 T403 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 127686 T403 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 127686 T403 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 127686 T403 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 127686 T403 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 127687 T403 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 127687 T403 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 127687 T403 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 127701 T403 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 127703 T403 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 127704 T403 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 127706 T403 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 127713 T403 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/plugins,/admin/segments,/admin/mbeans,/admin/ping,/update/csv,/update/json,/get,/config,/admin/threads,standard,/replication,/admin/logging,/admin/properties,/update,/update/json/docs,/admin/file,/admin/luke,/admin/system,/schema
   [junit4]   2> 127714 T403 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 127715 T403 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 127716 T403 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 127717 T403 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 127717 T403 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 127718 T403 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=15, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6488594503462402]
   [junit4]   2> 127719 T403 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1dd9940f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@37877418),segFN=segments_1,generation=1}
   [junit4]   2> 127719 T403 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 127719 T403 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@746186b1[collection1] main
   [junit4]   2> 127719 T403 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 127721 T403 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 127721 T403 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 127721 T403 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 127721 T403 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 127722 T403 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 127722 T403 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 127722 T403 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 127723 T403 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 127723 T403 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 127724 T404 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@746186b1[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 127725 T403 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 127725 T407 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:23333/i_zk/cc collection:collection1 shard:shard1
   [junit4]   2> 127726 T346 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 127726 T346 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 127726 T407 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 127732 T407 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 127734 T372 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 127734 T407 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 127734 T407 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C39 name=collection1 org.apache.solr.core.SolrCore@cde2a73 url=http://127.0.0.1:23333/i_zk/cc/collection1 node=127.0.0.1:23333_i_zk%2Fcc C39_STATE=coll:collection1 core:collection1 props:{node_name=127.0.0.1:23333_i_zk%2Fcc, state=down, core=collection1, base_url=http://127.0.0.1:23333/i_zk/cc}
   [junit4]   2> 127735 T407 C:collection1 S:shard1 C39 P23333 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:23333/i_zk/cc/collection1/
   [junit4]   2> 127735 T407 C:collection1 S:shard1 C39 P23333 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 127735 T373 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 4
   [junit4]   2> 127735 T407 C:collection1 S:shard1 C39 P23333 oasc.SyncStrategy.syncToMe http://127.0.0.1:23333/i_zk/cc/collection1/ has no replicas
   [junit4]   2> 127736 T407 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:23333/i_zk/cc/collection1/ shard1
   [junit4]   2> 127736 T407 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 127737 T373 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 127738 T402 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 127739 T402 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 127743 T373 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23333/i_zk/cc",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 4
   [junit4]   2> 127745 T373 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 127745 T402 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 127746 T402 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 127748 T372 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 127794 T407 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:23333/i_zk/cc/collection1/ and leader is http://127.0.0.1:23333/i_zk/cc/collection1/
   [junit4]   2> 127795 T407 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:23333/i_zk/cc
   [junit4]   2> 127795 T407 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 127795 T407 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 127795 T407 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 127797 T372 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 127798 T373 oasc.Overseer$ClusterStateUpdater.run 

[...truncated too long message...]

.ZkTestServer.send4LetterWord connecting to 127.0.0.1:16134 16134
   [junit4]   2> 1467795 T4297 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		4	/solr/collections/collection1/state.json
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1467795 T4296 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16199/
   [junit4]   2> 1467796 T4296 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16160/
   [junit4]   2> 1467796 T4296 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 12 connections to: http://127.0.0.1:16188/
   [junit4]   2> 1467796 T4296 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16214/
   [junit4]   2> 1467797 T4296 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:16230/
   [junit4]   2> 1467797 T4296 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16199/
   [junit4]   2> 1467798 T4296 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16160/
   [junit4]   2> 1467798 T4296 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16188/
   [junit4]   2> 1467798 T4296 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16214/
   [junit4]   2> 1467799 T4296 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16230/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=46171878E3F1D9B8 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=en_PH -Dtests.timezone=America/Iqaluit -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   77.1s J1 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16230/c8n_1x3_commits_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([46171878E3F1D9B8:CE4327A24D0DB440]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 48 more
   [junit4]   2> 1467819 T4296 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 46171878E3F1D9B8-001
   [junit4]   2> 77083 T4295 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=DefaultSimilarity, locale=en_PH, timezone=America/Iqaluit
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=139158576,total=333447168
   [junit4]   2> NOTE: All tests run in this JVM: [TestLeaderElectionZkExpiry, TestManagedSchema, TermsComponentTest, CopyFieldTest, MinimalSchemaTest, SolrIndexSplitterTest, CursorPagingTest, TestClassNameShortening, TestFastLRUCache, TestMiniSolrCloudClusterSSL, RequestLoggingTest, SpellCheckComponentTest, HighlighterMaxOffsetTest, BadIndexSchemaTest, CoreMergeIndexesAdminHandlerTest, RecoveryAfterSoftCommitTest, CSVRequestHandlerTest, FastVectorHighlighterTest, TestSortingResponseWriter, CustomCollectionTest, DistribCursorPagingTest, AddBlockUpdateTest, BasicFunctionalityTest, OverseerRolesTest, TestWordDelimiterFilterFactory, SpatialRPTFieldTypeTest, TestLazyCores, DistributedExpandComponentTest, TestReplicationHandler, SuggesterWFSTTest, UniqFieldsUpdateProcessorFactoryTest, TestRecovery, TestConfigSets, TestCoreDiscovery, RequestHandlersTest, LeaderFailoverAfterPartitionTest, PolyFieldTest, TestCollationField, TestManagedSchemaFieldResource, TestStressReorder, TestStressRecovery, TestBulkSchemaConcurrent, TestJettySolrRunner, TestComponentsName, TestSolrDynamicMBean, TestCharFilters, OverseerCollectionProcessorTest, TestDistributedGrouping, TestExceedMaxTermLength, SearchHandlerTest, SpellingQueryConverterTest, TestPartialUpdateDeduplication, TimeZoneUtilsTest, JsonLoaderTest, TestRealTimeGet, LukeRequestHandlerTest, TestSweetSpotSimilarityFactory, TestLMJelinekMercerSimilarityFactory, AliasIntegrationTest, FullSolrCloudDistribCmdsTest, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J1 in 78.13s, 1 test, 1 error <<< FAILURES!

[...truncated 695 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 52 minutes 36 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 213069433 bytes
Compression is 0.0%
Took 40 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2881 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2881/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:59379/e_/tc/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:59379/e_/tc/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([9B66993D745958DD:1332A6E7DAA53525]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:46914/ngb/h/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:46914/ngb/h/collection1
	at __randomizedtesting.SeedInfo.seed([9B66993D745958DD:1332A6E7DAA53525]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:46914/ngb/h/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19404/zt_/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19404/zt_/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([9B66993D745958DD:1332A6E7DAA53525]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19404/zt_/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9235 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/init-core-data-001
   [junit4]   2> 64852 T106 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /zt_/
   [junit4]   2> 64909 T106 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 64922 T107 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 65425 T106 oasc.ZkTestServer.run start zk server on port:21233
   [junit4]   2> 65641 T106 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 65994 T106 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 66226 T114 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1f39201f name:ZooKeeperConnection Watcher:127.0.0.1:21233 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 66227 T106 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 66227 T106 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 66242 T106 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 66405 T106 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 66406 T106 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 66407 T117 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@60874ad8 name:ZooKeeperConnection Watcher:127.0.0.1:21233/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 66408 T106 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 66408 T106 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 66419 T106 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 66423 T106 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 66426 T106 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 66428 T106 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 66432 T106 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 66433 T106 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 66438 T106 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 66438 T106 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 66442 T106 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 66442 T106 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 66445 T106 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 66446 T106 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 66449 T106 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 66449 T106 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 66453 T106 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 66453 T106 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 66457 T106 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 66457 T106 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 66460 T106 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 66461 T106 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 66464 T106 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 66465 T106 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 66468 T106 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 66468 T106 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 66471 T106 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 66472 T106 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 67729 T106 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/cores/collection1
   [junit4]   2> 67733 T106 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 67736 T106 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:21000
   [junit4]   2> 67736 T106 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/cores, hostPort=10802, hostContext=/zt_, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/tempDir-001/control/data}
   [junit4]   2> 67737 T106 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@5fb271f1
   [junit4]   2> 67737 T106 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/'
   [junit4]   2> 67769 T106 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/solr.xml
   [junit4]   2> 67832 T106 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/cores
   [junit4]   2> 67833 T106 oasc.CoreContainer.<init> New CoreContainer 14898748
   [junit4]   2> 67833 T106 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/]
   [junit4]   2> 67834 T106 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/lib
   [junit4]   2> 67834 T106 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/lib).
   [junit4]   2> 67846 T106 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 67846 T106 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 67846 T106 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 67847 T106 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 67847 T106 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 67847 T106 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 67847 T106 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 67847 T106 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 67848 T106 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 67848 T106 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 67848 T106 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 67849 T106 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 67849 T106 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 67849 T106 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 67850 T106 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 67850 T106 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:21233/solr
   [junit4]   2> 67884 T106 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 67885 T106 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 67885 T106 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 67889 T129 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1eaa58ff name:ZooKeeperConnection Watcher:127.0.0.1:21233 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 67890 T106 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 67891 T106 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 67905 T106 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 67907 T132 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@165545e8 name:ZooKeeperConnection Watcher:127.0.0.1:21233/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 67907 T106 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 67929 T106 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 67946 T106 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 67958 T106 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 67962 T106 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 67966 T106 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 67995 T106 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 67999 T106 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 68002 T106 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 68004 T106 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:10802_zt_
   [junit4]   2> 68013 T106 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:10802_zt_
   [junit4]   2> 68017 T106 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 68020 T106 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 68023 T106 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 68053 T106 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:10802_zt_
   [junit4]   2> 68054 T106 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 68057 T106 oasc.Overseer.start Overseer (id=93587679924649987-127.0.0.1:10802_zt_-n_0000000000) starting
   [junit4]   2> 68079 T106 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 68125 T106 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 68147 T134 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 68147 T106 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 68171 T133 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 68183 T106 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/cores
   [junit4]   2> 68198 T106 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/cores/collection1, collection=control_collection, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/cores/collection1/, config=solrconfig.xml, transient=false, name=collection1, dataDir=data/, coreNodeName=, shard=, loadOnStartup=true, schema=schema.xml}
   [junit4]   2> 68199 T106 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/cores/collection1/
   [junit4]   2> 68199 T106 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 68201 T136 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 68201 T136 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 68219 T132 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 68219 T136 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 68225 T133 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:10802_zt_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:10802/zt_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"} current state version: 0
   [junit4]   2> 68263 T133 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:10802_zt_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:10802/zt_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 68269 T133 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 68309 T133 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 68324 T132 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 69221 T136 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 69221 T136 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 69223 T136 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 69223 T136 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 69225 T136 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 69226 T136 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/cores/collection1/'
   [junit4]   2> 69317 T136 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 69357 T136 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 69410 T136 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 69452 T136 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 69532 T136 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 69534 T136 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 69558 T136 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 70139 T136 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 70164 T136 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 70170 T136 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 70243 T136 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 70250 T136 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 70257 T136 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 70259 T136 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 70259 T136 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 70259 T136 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 70261 T136 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 70261 T136 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 70262 T136 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 70262 T136 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 70263 T136 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/cores/collection1/data/
   [junit4]   2> 70263 T136 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@15185382
   [junit4]   2> 70284 T136 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/cores/collection1/data
   [junit4]   2> 70284 T136 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/cores/collection1/data/index/
   [junit4]   2> 70284 T136 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 70285 T136 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/cores/collection1/data/index
   [junit4]   2> 70285 T136 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=35, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 70287 T136 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@57cea843 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4088f3be),segFN=segments_1,generation=1}
   [junit4]   2> 70287 T136 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 70301 T136 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 70302 T136 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 70302 T136 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 70302 T136 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 70303 T136 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 70303 T136 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 70304 T136 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 70304 T136 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 70304 T136 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 70329 T136 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 70341 T136 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 70345 T136 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 70349 T136 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 70365 T136 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/segments,/admin/ping,/admin/mbeans,/admin/logging,/get,/admin/system,/admin/luke,/admin/properties,/admin/plugins,standard,/schema,/update,/update/json,/update/json/docs,/admin/file,/replication,/update/csv,/config,/admin/threads
   [junit4]   2> 70368 T136 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 70370 T136 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 70370 T136 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 70371 T136 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 70372 T136 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 70373 T136 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=31, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8930320413430614]
   [junit4]   2> 70374 T136 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@57cea843 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4088f3be),segFN=segments_1,generation=1}
   [junit4]   2> 70374 T136 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 70375 T136 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@5854278[collection1] main
   [junit4]   2> 70375 T136 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 70377 T136 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 70377 T136 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 70390 T136 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 70391 T136 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 70391 T136 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 70392 T136 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 70393 T136 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 70393 T136 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 70393 T136 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 70397 T137 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5854278[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 70406 T136 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 70407 T140 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:10802/zt_ collection:control_collection shard:shard1
   [junit4]   2> 70408 T106 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 70409 T106 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 70448 T140 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 70459 T140 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 70463 T132 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 70464 T140 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 70464 T140 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 70464 T133 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C16 name=collection1 org.apache.solr.core.SolrCore@3cbbaaf4 url=http://127.0.0.1:10802/zt_/collection1 node=127.0.0.1:10802_zt_ C16_STATE=coll:control_collection core:collection1 props:{state=down, node_name=127.0.0.1:10802_zt_, base_url=http://127.0.0.1:10802/zt_, core=collection1}
   [junit4]   2> 70473 T140 C:control_collection S:shard1 C16 P10802 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:10802/zt_/collection1/
   [junit4]   2> 70474 T140 C:control_collection S:shard1 C16 P10802 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 70474 T140 C:control_collection S:shard1 C16 P10802 oasc.SyncStrategy.syncToMe http://127.0.0.1:10802/zt_/collection1/ has no replicas
   [junit4]   2> 70475 T140 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:10802/zt_/collection1/ shard1
   [junit4]   2> 70476 T106 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 70477 T106 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 70479 T132 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 70480 T144 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5c14f4c name:ZooKeeperConnection Watcher:127.0.0.1:21233/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 70480 T106 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 70482 T106 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 70482 T106 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 70490 T140 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 70491 T106 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 70497 T132 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 70499 T133 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:10802/zt_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 70606 T144 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 70606 T132 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 70652 T140 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:10802/zt_/collection1/ and leader is http://127.0.0.1:10802/zt_/collection1/
   [junit4]   2> 70652 T140 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:10802/zt_
   [junit4]   2> 70652 T140 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 70653 T140 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 70653 T140 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 70656 T132 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 70658 T133 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:10802_zt_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:10802/zt_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"} current state version: 3
   [junit4]   2> 70659 T133 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:10802_zt_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:10802/zt_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 70766 T132 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 70766 T144 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 71975 T106 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/cores/collection1
   [junit4]   2> 71978 T106 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001
   [junit4]   2> 71978 T106 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 71981 T106 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:35143
   [junit4]   2> 71981 T106 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/cores, solrconfig=solrconfig.xml, hostPort=13397, hostContext=/zt_, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/tempDir-001/jetty1}
   [junit4]   2> 71982 T106 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@5fb271f1
   [junit4]   2> 71982 T106 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/'
   [junit4]   2> 72013 T106 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/solr.xml
   [junit4]   2> 72053 T106 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/cores
   [junit4]   2> 72053 T106 oasc.CoreContainer.<init> New CoreContainer 2009355367
   [junit4]   2> 72053 T106 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/]
   [junit4]   2> 72054 T106 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/lib
   [junit4]   2> 72054 T106 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/lib).
   [junit4]   2> 72064 T106 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 72064 T106 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 72065 T106 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 72065 T106 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 72065 T106 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 72065 T106 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 72065 T106 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 72066 T106 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 72066 T106 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 72066 T106 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 72066 T106 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 72067 T106 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 72067 T106 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 72068 T106 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 72068 T106 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 72068 T106 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:21233/solr
   [junit4]   2> 72068 T106 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 72069 T106 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 72069 T106 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 72071 T156 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@35896b6 name:ZooKeeperConnection Watcher:127.0.0.1:21233 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 72071 T106 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 72072 T106 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 72074 T106 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 72076 T159 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7e308c04 name:ZooKeeperConnection Watcher:127.0.0.1:21233/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 72076 T106 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 72082 T106 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 73090 T106 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:13397_zt_
   [junit4]   2> 73092 T106 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:13397_zt_
   [junit4]   2> 73097 T106 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 73107 T106 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/cores
   [junit4]   2> 73110 T106 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, collection=collection1, loadOnStartup=true, schema=schema.xml, dataDir=data/, name=collection1, coreNodeName=, transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/cores/collection1/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/cores/collection1, config=solrconfig.xml}
   [junit4]   2> 73110 T106 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/cores/collection1/
   [junit4]   2> 73111 T106 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 73112 T160 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 73112 T160 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 73114 T132 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 73114 T160 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 73115 T133 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:13397_zt_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:13397/zt_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"} current state version: 4
   [junit4]   2> 73116 T133 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:13397_zt_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:13397/zt_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 73117 T133 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 73117 T133 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 73120 T159 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 73120 T132 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 73120 T144 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 74114 T160 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 74115 T160 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 74116 T160 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 74116 T160 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 74118 T160 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 74118 T160 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/cores/collection1/'
   [junit4]   2> 74138 T160 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 74141 T160 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 74156 T160 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 74184 T160 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 74264 T160 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 74267 T160 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 74289 T160 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 74788 T160 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 74810 T160 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 74814 T160 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 74845 T160 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 74852 T160 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 74858 T160 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 74860 T160 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 74860 T160 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 74860 T160 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 74862 T160 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 74863 T160 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 74863 T160 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 74863 T160 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 74864 T160 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 74864 T160 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@15185382
   [junit4]   2> 74865 T160 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/cores/collection1/data
   [junit4]   2> 74866 T160 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 74866 T160 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 74867 T160 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 74867 T160 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=35, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 74868 T160 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@49f45c33 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@49d13e33),segFN=segments_1,generation=1}
   [junit4]   2> 74869 T160 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 74884 T160 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 74885 T160 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 74885 T160 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 74886 T160 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 74886 T160 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 74886 T160 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 74887 T160 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 74887 T160 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 74887 T160 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 74910 T160 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 74914 T160 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 74917 T160 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 74921 T160 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 74960 T160 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/segments,/admin/ping,/admin/mbeans,/admin/logging,/get,/admin/system,/admin/luke,/admin/properties,/admin/plugins,standard,/schema,/update,/update/json,/update/json/docs,/admin/file,/replication,/update/csv,/config,/admin/threads
   [junit4]   2> 74963 T160 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 74964 T160 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 74965 T160 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 74966 T160 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 74967 T160 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 74969 T160 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=31, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8930320413430614]
   [junit4]   2> 74971 T160 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@49f45c33 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@49d13e33),segFN=segments_1,generation=1}
   [junit4]   2> 74971 T160 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 74971 T160 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@1c414bd8[collection1] main
   [junit4]   2> 74972 T160 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 74974 T160 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 74974 T160 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 74975 T160 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 74975 T160 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 74975 T160 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 74976 T160 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 74977 T160 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 74977 T160 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 74977 T160 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 74980 T161 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1c414bd8[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 74981 T160 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 74982 T164 C:collection1 S:shard3 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:13397/zt_ collection:collection1 shard:shard3
   [junit4]   2> 74983 T106 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 74984 T106 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 74984 T164 C:collection1 S:shard3 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
   [junit4]   2> 74994 T164 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard3
   [junit4]   2> 74997 T132 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 74998 T164 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 74998 T164 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C17 name=collection1 org.apache.solr.core.SolrCore@7f17f2e0 url=http://127.0.0.1:13397/zt_/collection1 node=127.0.0.1:13397_zt_ C17_STATE=coll:collection1 core:collection1 props:{state=down, node_name=127.0.0.1:13397_zt_, base_url=http://127.0.0.1:13397/zt_, core=collection1}
   [junit4]   2> 74998 T164 C:collection1 S:shard3 C17 P13397 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:13397/zt_/collection1/
   [junit4]   2> 74999 T164 C:collection1 S:shard3 C17 P13397 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 74999 T164 C:collection1 S:shard3 C17 P13397 oasc.SyncStrategy.syncToMe http://127.0.0.1:13397/zt_/collection1/ has no replicas
   [junit4]   2> 74999 T133 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 75000 T164 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:13397/zt_/collection1/ shard3
   [junit4]   2> 75000 T164 C:collection1 S:shard3 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard3
   [junit4]   2> 75003 T144 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 75003 T132 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 75003 T159 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 75011 T133 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:13397/zt_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 75015 T132 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 75017 T159 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 75017 T132 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 75017 T144 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 75062 T164 C:collection1 S:shard3 oasc.ZkController.register We are http://127.0.0.1:13397/zt_/collection1/ and leader is http://127.0.0.1:13397/zt_/collection1/
   [junit4]   2> 75063 T164 C:collection1 S:shard3 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:13397/zt_
   [junit4]   2> 75063 T164 C:collection1 S:shard3 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 75063 T164 C:collection1 S:shard3 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 75063 T164 C:collection1 S:shard3 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 75066 T132 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 75068 T133 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:13397_zt_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:13397/zt_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"} current state version: 7
   [junit4]   2> 75069 T133 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:13397_zt_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:13397/zt_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 75177 T144 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 75177 T132 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 75177 T159 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 76699 T106 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-2-001/cores/collection1
   [junit4]   2> 76703 T106 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-2-001
   [junit4]   2> 76704 T106 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 76707 T106 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:38060


[...truncated too long message...]

til.java:31)
   [junit4]   2> 		at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:142)
   [junit4]   2> 		... 11 more
   [junit4]   2> 	
   [junit4]   2> 852947 T2693 C:control_collection S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/e_/tc,null}
   [junit4]   2> 853041 T2693 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:59358 59358
   [junit4]   2> 853381 T2694 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:59358 59358
   [junit4]   2> 853382 T2694 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 853383 T2693 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:59365/e_/tc
   [junit4]   2> 853383 T2693 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:59426/e_/tc
   [junit4]   2> 853383 T2693 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:59391/e_/tc
   [junit4]   2> 853383 T2693 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 14 connections to: http://127.0.0.1:59371/e_/tc
   [junit4]   2> 853384 T2693 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:59379/e_/tc
   [junit4]   2> 853384 T2693 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:59365/e_/tc
   [junit4]   2> 853384 T2693 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:59426/e_/tc
   [junit4]   2> 853384 T2693 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:59391/e_/tc
   [junit4]   2> 853384 T2693 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:59371/e_/tc
   [junit4]   2> 853385 T2693 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:59379/e_/tc
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=9B66993D745958DD -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=cs_CZ -Dtests.timezone=Europe/Malta -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   83.5s J1 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:59379/e_/tc/c8n_1x3_commits_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([9B66993D745958DD:1332A6E7DAA53525]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 48 more
   [junit4]   2> 853391 T2693 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 9B66993D745958DD-001
   [junit4]   2> 83493 T2692 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=DefaultSimilarity, locale=cs_CZ, timezone=Europe/Malta
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=103965640,total=296747008
   [junit4]   2> NOTE: All tests run in this JVM: [SpatialRPTFieldTypeTest, MigrateRouteKeyTest, TestReRankQParserPlugin, CustomCollectionTest, JsonLoaderTest, TestPostingsSolrHighlighter, RequestLoggingTest, TestClassNameShortening, TestFieldSortValues, DirectUpdateHandlerTest, OpenExchangeRatesOrgProviderTest, TestManagedSchemaFieldTypeResource, TermVectorComponentTest, RecoveryAfterSoftCommitTest, TestSolrQueryParserResource, MoreLikeThisHandlerTest, AnalysisErrorHandlingTest, ZkStateWriterTest, TestSchemaResource, TestManagedStopFilterFactory, InfoHandlerTest, TestMiniSolrCloudClusterSSL, BadComponentTest, TestStandardQParsers, HdfsRecoveryZkTest, CollectionsAPIDistributedZkTest, ChaosMonkeyNothingIsSafeTest, TestLFUCache, EchoParamsTest, SolrCoreTest, DistribJoinFromCollectionTest, UUIDFieldTest, ExternalFileFieldSortTest, TestUniqueKeyFieldResource, TestDistribDocBasedVersion, TestRandomFaceting, TestManagedSchema, FileBasedSpellCheckerTest, IndexSchemaTest, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J1 in 83.77s, 1 test, 1 error <<< FAILURES!

[...truncated 867 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 56 minutes 56 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 206578940 bytes
Compression is 0.0%
Took 44 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2880 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2880/

4 tests failed.
REGRESSION:  org.apache.solr.cloud.RecoveryZkTest.test

Error Message:
shard1 is not consistent.  Got 501 from http://127.0.0.1:39753/mkrd/collection1lastClient and got 264 from http://127.0.0.1:39784/mkrd/collection1

Stack Trace:
java.lang.AssertionError: shard1 is not consistent.  Got 501 from http://127.0.0.1:39753/mkrd/collection1lastClient and got 264 from http://127.0.0.1:39784/mkrd/collection1
	at __randomizedtesting.SeedInfo.seed([5E8BB946415F99B4:D6DF869CEFA3F44C]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:123)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:39306/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:39306/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([5E8BB946415F99B4:D6DF869CEFA3F44C]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:31114/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:31114/collection1
	at __randomizedtesting.SeedInfo.seed([5E8BB946415F99B4:D6DF869CEFA3F44C]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:31114/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:56201/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:56201/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([5E8BB946415F99B4:D6DF869CEFA3F44C]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:56201/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9464 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/init-core-data-001
   [junit4]   2> 269650 T884 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 269655 T884 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 269656 T885 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 269756 T884 oasc.ZkTestServer.run start zk server on port:31099
   [junit4]   2> 269757 T884 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 269758 T884 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 269761 T892 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@657ecaa0 name:ZooKeeperConnection Watcher:127.0.0.1:31099 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 269761 T884 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 269761 T884 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 269762 T884 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 269764 T884 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 269765 T884 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 269766 T895 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5b0f66d6 name:ZooKeeperConnection Watcher:127.0.0.1:31099/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 269766 T884 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 269766 T884 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 269766 T884 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 269768 T884 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 269769 T884 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 269770 T884 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 269772 T884 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 269772 T884 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 269775 T884 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 269775 T884 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 269777 T884 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 269777 T884 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 269779 T884 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 269779 T884 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 269781 T884 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 269781 T884 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 269783 T884 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 269783 T884 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 269785 T884 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 269785 T884 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 269787 T884 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 269787 T884 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 269789 T884 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 269789 T884 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 269791 T884 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 269792 T884 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 269793 T884 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 269794 T884 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 271412 T884 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/cores/collection1
   [junit4]   2> 271416 T884 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 271420 T884 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:31103
   [junit4]   2> 271420 T884 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/tempDir-001/control/data, hostPort=31102, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/cores}
   [junit4]   2> 271421 T884 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3972aa3f
   [junit4]   2> 271421 T884 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/'
   [junit4]   2> 271469 T884 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/solr.xml
   [junit4]   2> 271529 T884 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/cores
   [junit4]   2> 271530 T884 oasc.CoreContainer.<init> New CoreContainer 914690658
   [junit4]   2> 271530 T884 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/]
   [junit4]   2> 271530 T884 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/lib
   [junit4]   2> 271531 T884 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/lib).
   [junit4]   2> 271541 T884 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 271541 T884 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 271542 T884 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 271542 T884 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 271542 T884 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 271542 T884 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 271542 T884 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 271543 T884 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 271543 T884 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 271543 T884 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 271543 T884 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 271544 T884 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 271544 T884 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 271544 T884 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 271544 T884 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 271545 T884 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:31099/solr
   [junit4]   2> 271545 T884 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 271545 T884 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 271546 T884 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 271547 T907 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@49765877 name:ZooKeeperConnection Watcher:127.0.0.1:31099 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 271548 T884 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 271548 T884 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 271550 T884 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 271551 T910 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@28fa04f3 name:ZooKeeperConnection Watcher:127.0.0.1:31099/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 271551 T884 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 271553 T884 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 271555 T884 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 271556 T884 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 271558 T884 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 271560 T884 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 271562 T884 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 271563 T884 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 271564 T884 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 271565 T884 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:31102_
   [junit4]   2> 271566 T884 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:31102_
   [junit4]   2> 271567 T884 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 271568 T884 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 271570 T884 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 271571 T884 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:31102_
   [junit4]   2> 271571 T884 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 271572 T884 oasc.Overseer.start Overseer (id=93586741821046787-127.0.0.1:31102_-n_0000000000) starting
   [junit4]   2> 271574 T884 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 271579 T884 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 271580 T912 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 271580 T884 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 271580 T911 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 271583 T884 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/cores
   [junit4]   2> 271585 T884 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {transient=false, config=solrconfig.xml, dataDir=data/, name=collection1, coreNodeName=, loadOnStartup=true, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/cores/collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/cores/collection1/, collection=control_collection, shard=, schema=schema.xml}
   [junit4]   2> 271586 T884 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/cores/collection1/
   [junit4]   2> 271586 T884 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 271587 T914 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 271587 T914 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 271588 T910 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 271588 T914 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 271589 T911 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:31102",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:31102_",
   [junit4]   2> 	  "state":"down"} current state version: 0
   [junit4]   2> 271589 T911 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:31102",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:31102_",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 271589 T911 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 271589 T911 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 271590 T910 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 272589 T914 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 272590 T914 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 272590 T914 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 272590 T914 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 272591 T914 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 272591 T914 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/cores/collection1/'
   [junit4]   2> 272605 T914 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 272607 T914 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 272618 T914 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 272638 T914 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 272819 T914 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 272820 T914 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 272829 T914 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 273176 T914 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 273192 T914 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 273194 T914 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 273203 T914 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 273207 T914 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 273210 T914 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 273211 T914 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 273211 T914 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 273211 T914 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 273212 T914 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 273212 T914 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 273212 T914 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 273212 T914 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 273212 T914 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/cores/collection1/data/
   [junit4]   2> 273213 T914 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a27d7d0
   [junit4]   2> 273213 T914 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/cores/collection1/data
   [junit4]   2> 273213 T914 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/cores/collection1/data/index/
   [junit4]   2> 273214 T914 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 273214 T914 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/cores/collection1/data/index
   [junit4]   2> 273214 T914 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=39, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8675328596970137]
   [junit4]   2> 273215 T914 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@37bf1383 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@619a1a0c),segFN=segments_1,generation=1}
   [junit4]   2> 273215 T914 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 273219 T914 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 273219 T914 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 273219 T914 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 273219 T914 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 273219 T914 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 273220 T914 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 273220 T914 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 273220 T914 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 273220 T914 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 273247 T914 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 273249 T914 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 273251 T914 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 273253 T914 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 273256 T914 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/csv,/admin/system,/update/json/docs,/admin/plugins,/admin/luke,/replication,standard,/get,/admin/mbeans,/admin/logging,/admin/segments,/update/json,/admin/threads,/config,/update,/admin/ping,/admin/file,/admin/properties,/schema
   [junit4]   2> 273258 T914 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 273260 T914 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 273260 T914 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 273262 T914 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 273262 T914 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 273262 T914 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=48, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7866241976657329]
   [junit4]   2> 273263 T914 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@37bf1383 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@619a1a0c),segFN=segments_1,generation=1}
   [junit4]   2> 273263 T914 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 273263 T914 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@35a6ab1d[collection1] main
   [junit4]   2> 273263 T914 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 273264 T914 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 273264 T914 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 273265 T914 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 273265 T914 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 273265 T914 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 273265 T914 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 273265 T914 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 273266 T914 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 273266 T914 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 273266 T915 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@35a6ab1d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 273267 T914 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 273267 T918 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:31102 collection:control_collection shard:shard1
   [junit4]   2> 273267 T884 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 273268 T884 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 273268 T918 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 273271 T884 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 273271 T918 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 273272 T884 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 273273 T922 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@665b8cbf name:ZooKeeperConnection Watcher:127.0.0.1:31099/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 273273 T884 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 273273 T910 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 273273 T884 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 273273 T918 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 273273 T884 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 273274 T911 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 273274 T918 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C194 name=collection1 org.apache.solr.core.SolrCore@52c8671d url=http://127.0.0.1:31102/collection1 node=127.0.0.1:31102_ C194_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:31102, core=collection1, node_name=127.0.0.1:31102_, state=down}
   [junit4]   2> 273275 T918 C:control_collection S:shard1 C194 P31102 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:31102/collection1/
   [junit4]   2> 273275 T918 C:control_collection S:shard1 C194 P31102 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 273275 T918 C:control_collection S:shard1 C194 P31102 oasc.SyncStrategy.syncToMe http://127.0.0.1:31102/collection1/ has no replicas
   [junit4]   2> 273275 T918 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:31102/collection1/ shard1
   [junit4]   2> 273275 T918 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 273275 T910 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 273275 T922 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 273277 T884 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 273279 T910 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 273279 T911 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:31102",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 273382 T910 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 273382 T922 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 273432 T918 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:31102/collection1/ and leader is http://127.0.0.1:31102/collection1/
   [junit4]   2> 273433 T918 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:31102
   [junit4]   2> 273433 T918 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 273433 T918 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 273433 T918 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 273434 T910 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 273435 T911 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:31102",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:31102_",
   [junit4]   2> 	  "state":"active"} current state version: 3
   [junit4]   2> 273436 T911 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:31102",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:31102_",
   [junit4]   2> 	  "state":"active"}
   [junit4]   2> 273539 T910 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 273539 T922 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 274649 T884 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/cores/collection1
   [junit4]   2> 274652 T884 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001
   [junit4]   2> 274653 T884 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 274656 T884 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:31115
   [junit4]   2> 274656 T884 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/cores, hostContext=/, hostPort=31114, solrconfig=solrconfig.xml}
   [junit4]   2> 274656 T884 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3972aa3f
   [junit4]   2> 274657 T884 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/'
   [junit4]   2> 274711 T884 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/solr.xml
   [junit4]   2> 274784 T884 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/cores
   [junit4]   2> 274784 T884 oasc.CoreContainer.<init> New CoreContainer 540568750
   [junit4]   2> 274785 T884 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/]
   [junit4]   2> 274785 T884 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/lib
   [junit4]   2> 274786 T884 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/lib).
   [junit4]   2> 274798 T884 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 274799 T884 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 274799 T884 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 274799 T884 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 274799 T884 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 274800 T884 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 274800 T884 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 274800 T884 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 274800 T884 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 274801 T884 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 274801 T884 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 274802 T884 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 274802 T884 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 274802 T884 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 274802 T884 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 274803 T884 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:31099/solr
   [junit4]   2> 274803 T884 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 274803 T884 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 274804 T884 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 274806 T934 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4e178d9c name:ZooKeeperConnection Watcher:127.0.0.1:31099 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 274806 T884 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 274807 T884 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 274809 T884 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 274810 T937 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@335b4293 name:ZooKeeperConnection Watcher:127.0.0.1:31099/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 274810 T884 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 274815 T884 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 275820 T884 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:31114_
   [junit4]   2> 275821 T884 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:31114_
   [junit4]   2> 275823 T884 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 275825 T884 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/cores
   [junit4]   2> 275827 T884 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {dataDir=data/, loadOnStartup=true, config=solrconfig.xml, name=collection1, schema=schema.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/cores/collection1/, transient=false, collection=collection1, shard=, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/cores/collection1}
   [junit4]   2> 275828 T884 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/cores/collection1/
   [junit4]   2> 275828 T884 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 275829 T938 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 275829 T938 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 275829 T938 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 275829 T910 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 275830 T911 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:31114",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:31114_",
   [junit4]   2> 	  "state":"down"} current state version: 4
   [junit4]   2> 275831 T911 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:31114",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:31114_",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 275831 T911 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 275831 T911 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 275832 T922 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 275832 T937 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 275832 T910 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 276831 T938 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 276832 T938 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 276832 T938 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 276832 T938 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 276833 T938 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 276833 T938 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/cores/collection1/'
   [junit4]   2> 276846 T938 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 276848 T938 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 276858 T938 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 276877 T938 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 276917 T938 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 276918 T938 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 276927 T938 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 277266 T938 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 277285 T938 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 277289 T938 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 277301 T938 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 277305 T938 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 277310 T938 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 277310 T938 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 277311 T938 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 277311 T938 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 277312 T938 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 277312 T938 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 277312 T938 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 277313 T938 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 277313 T938 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 277313 T938 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a27d7d0
   [junit4]   2> 277314 T938 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/cores/collection1/data
   [junit4]   2> 277314 T938 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 277315 T938 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 277315 T938 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 277315 T938 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=39, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8675328596970137]
   [junit4]   2> 277316 T938 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3f1bc613 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@386929c3),segFN=segments_1,generation=1}
   [junit4]   2> 277316 T938 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 277323 T938 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 277324 T938 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 277324 T938 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 277324 T938 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 277325 T938 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 277325 T938 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 277326 T938 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 277326 T938 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 277326 T938 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 277356 T938 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 277358 T938 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 277361 T938 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 277364 T938 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 277372 T938 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/csv,/admin/system,/update/json/docs,/admin/plugins,/admin/luke,/replication,standard,/get,/admin/mbeans,/admin/logging,/admin/segments,/update/json,/admin/threads,/config,/update,/admin/ping,/admin/file,/admin/properties,/schema
   [junit4]   2> 277374 T938 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 277376 T938 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 277376 T938 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 277378 T938 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 277378 T938 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 277379 T938 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=48, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7866241976657329]
   [junit4]   2> 277380 T938 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3f1bc613 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@386929c3),segFN=segments_1,generation=1}
   [junit4]   2> 277380 T938 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 277381 T938 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@71971b44[collection1] main
   [junit4]   2> 277381 T938 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 277382 T938 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 277382 T938 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 277382 T938 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 277383 T938 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 277383 T938 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 277383 T938 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 277384 T938 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 277384 T938 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 277384 T938 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 277385 T939 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@71971b44[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 277386 T938 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 277386 T942 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:31114 collection:collection1 shard:shard1
   [junit4]   2> 277387 T884 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 277387 T884 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 277387 T942 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 277392 T942 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 277393 T910 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 277393 T942 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 277394 T942 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C195 name=collection1 org.apache.solr.core.SolrCore@761f2384 url=http://127.0.0.1:31114/collection1 node=127.0.0.1:31114_ C195_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:31114, core=collection1, node_name=127.0.0.1:31114_, state=down}
   [junit4]   2> 277394 T942 C:collection1 S:shard1 C195 P31114 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:31114/collection1/
   [junit4]   2> 277394 T911 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 277394 T942 C:collection1 S:shard1 C195 P31114 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 277394 T942 C:collection1 S:shard1 C195 P31114 oasc.SyncStrategy.syncToMe http://127.0.0.1:31114/collection1/ has no replicas
   [junit4]   2> 277395 T942 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:31114/collection1/ sh

[...truncated too long message...]

13 T3106 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:39267 39267
   [junit4]   2> 1478809 T3107 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:39267 39267
   [junit4]   2> 1478810 T3107 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/queue
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1478811 T3106 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:39278/
   [junit4]   2> 1478811 T3106 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:39360/
   [junit4]   2> 1478811 T3106 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 3 connections to: http://127.0.0.1:39318/
   [junit4]   2> 1478811 T3106 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 13 connections to: http://127.0.0.1:39306/
   [junit4]   2> 1478812 T3106 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:39326/
   [junit4]   2> 1478812 T3106 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:39278/
   [junit4]   2> 1478812 T3106 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:39360/
   [junit4]   2> 1478812 T3106 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:39318/
   [junit4]   2> 1478813 T3106 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:39306/
   [junit4]   2> 1478813 T3106 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:39326/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=5E8BB946415F99B4 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=es_BO -Dtests.timezone=IET -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   83.2s J0 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:39306/c8n_1x3_commits_shard1_replica3
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([5E8BB946415F99B4:D6DF869CEFA3F44C]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 48 more
   [junit4]   2> 1478826 T3106 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 5E8BB946415F99B4-001
   [junit4]   2> 83169 T3105 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=DefaultSimilarity, locale=es_BO, timezone=IET
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=155916320,total=346030080
   [junit4]   2> NOTE: All tests run in this JVM: [JsonLoaderTest, StressHdfsTest, DistributedFacetPivotLargeTest, DateMathParserTest, CurrencyFieldOpenExchangeTest, DistribJoinFromCollectionTest, FacetPivotSmallTest, TestLRUCache, TestHighlightDedupGrouping, OverseerCollectionProcessorTest, AtomicUpdatesTest, TestSweetSpotSimilarityFactory, TestSimpleTrackingShardHandler, BasicZkTest, SolrIndexSplitterTest, TestCSVResponseWriter, TestSchemaManager, SolrTestCaseJ4Test, TestRandomFaceting, TestBadConfig, CloudMLTQParserTest, TestStressVersions, NumericFieldsTest, TestReplicationHandlerBackup, QueryResultKeyTest, DateFieldTest, TestRTGBase, SpellingQueryConverterTest, TestWordDelimiterFilterFactory, ParsingFieldUpdateProcessorsTest, TestRecoveryHdfs, RecoveryAfterSoftCommitTest, CoreAdminCreateDiscoverTest, SearchHandlerTest, TestUpdate, TestAnalyzedSuggestions, TestPhraseSuggestions, TestRecovery, FastVectorHighlighterTest, SOLR749Test, TestFieldTypeResource, DocumentAnalysisRequestHandlerTest, TestPHPSerializedResponseWriter, ResponseHeaderTest, TestHashQParserPlugin, TestInitQParser, TestJettySolrRunner, SimpleFacetsTest, TestStressUserVersions, BadComponentTest, TestManagedResourceStorage, FullSolrCloudDistribCmdsTest, TestSchemaResource, MultiThreadedOCPTest, TestStressReorder, TestOmitPositions, TestExceedMaxTermLength, HttpPartitionTest, ShardSplitTest, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J0 in 83.71s, 1 test, 1 error <<< FAILURES!

[...truncated 722 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 1 failure, 61 ignored (21 assumptions)

Total time: 53 minutes 59 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 215712534 bytes
Compression is 0.0%
Took 40 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2879 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2879/

4 tests failed.
FAILED:  org.apache.solr.handler.component.DistributedMLTComponentTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:33912//collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:33912//collection1
	at __randomizedtesting.SeedInfo.seed([94BF5CA3001FF936:1CEB6379AEE394CE]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:566)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:558)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:606)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:588)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:567)
	at org.apache.solr.handler.component.DistributedMLTComponentTest.test(DistributedMLTComponentTest.java:126)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 51 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:57562/f/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:57562/f/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([94BF5CA3001FF936:1CEB6379AEE394CE]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:51163/p_myp/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:51163/p_myp/collection1
	at __randomizedtesting.SeedInfo.seed([94BF5CA3001FF936:1CEB6379AEE394CE]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:51163/p_myp/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:57725/z/su/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:57725/z/su/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([94BF5CA3001FF936:1CEB6379AEE394CE]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:57725/z/su/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9675 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/init-core-data-001
   [junit4]   2> 444750 T798 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /p_myp/
   [junit4]   2> 444754 T798 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 444755 T799 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 444855 T798 oasc.ZkTestServer.run start zk server on port:51149
   [junit4]   2> 444856 T798 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 444857 T798 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 444861 T806 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@71cc5eb5 name:ZooKeeperConnection Watcher:127.0.0.1:51149 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 444861 T798 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 444862 T798 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 444862 T798 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 444864 T798 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 444865 T798 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 444867 T809 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5611f639 name:ZooKeeperConnection Watcher:127.0.0.1:51149/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 444867 T798 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 444867 T798 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 444868 T798 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 444870 T798 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 444872 T798 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 444873 T798 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 444875 T798 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 444875 T798 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 444878 T798 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 444879 T798 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 444981 T798 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 444981 T798 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 444983 T798 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 444983 T798 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 444985 T798 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 444986 T798 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 444987 T798 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 444988 T798 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 444989 T798 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 444990 T798 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 444991 T798 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 444992 T798 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 444994 T798 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 444994 T798 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 444996 T798 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 444996 T798 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 444998 T798 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 444999 T798 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 446542 T798 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/cores/collection1
   [junit4]   2> 446546 T798 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 446549 T798 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:51154
   [junit4]   2> 446550 T798 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/p_myp, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/cores, hostPort=51153, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/tempDir-001/control/data}
   [junit4]   2> 446550 T798 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 446551 T798 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/'
   [junit4]   2> 446585 T798 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/solr.xml
   [junit4]   2> 446626 T798 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/cores
   [junit4]   2> 446626 T798 oasc.CoreContainer.<init> New CoreContainer 1742799638
   [junit4]   2> 446627 T798 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/]
   [junit4]   2> 446627 T798 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/lib
   [junit4]   2> 446627 T798 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/lib).
   [junit4]   2> 446637 T798 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 446637 T798 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 446637 T798 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 446638 T798 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 446638 T798 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 446638 T798 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 446638 T798 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 446638 T798 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 446639 T798 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 446639 T798 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 446639 T798 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 446640 T798 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 446640 T798 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 446640 T798 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 446640 T798 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 446640 T798 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51149/solr
   [junit4]   2> 446641 T798 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 446641 T798 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 446642 T798 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 446643 T821 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2d84ba1b name:ZooKeeperConnection Watcher:127.0.0.1:51149 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 446644 T798 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 446644 T798 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 446646 T798 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 446656 T824 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3b3019d2 name:ZooKeeperConnection Watcher:127.0.0.1:51149/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 446657 T798 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 446658 T798 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 446660 T798 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 446661 T798 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 446663 T798 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 446664 T798 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 446666 T798 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 446667 T798 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 446668 T798 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 446669 T798 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51153_p_myp
   [junit4]   2> 446670 T798 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51153_p_myp
   [junit4]   2> 446671 T798 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 446672 T798 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 446673 T798 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 446674 T798 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:51153_p_myp
   [junit4]   2> 446675 T798 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 446676 T798 oasc.Overseer.start Overseer (id=93586175975817219-127.0.0.1:51153_p_myp-n_0000000000) starting
   [junit4]   2> 446677 T798 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 446682 T798 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 446683 T826 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 446683 T798 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 446683 T825 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 446686 T798 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/cores
   [junit4]   2> 446688 T798 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, collection=control_collection, dataDir=data/, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/cores/collection1, shard=, coreNodeName=, name=collection1, loadOnStartup=true, transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/cores/collection1/}
   [junit4]   2> 446689 T798 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/cores/collection1/
   [junit4]   2> 446689 T798 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 446689 T828 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 446690 T828 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 446690 T828 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 446690 T824 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 446691 T825 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:51153_p_myp",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51153/p_myp",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null} current state version: 0
   [junit4]   2> 446691 T825 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:51153_p_myp",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51153/p_myp",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 446692 T825 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 446692 T825 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 446693 T824 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 447691 T828 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 447692 T828 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 447692 T828 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 447693 T828 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 447693 T828 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 447693 T828 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/cores/collection1/'
   [junit4]   2> 447705 T828 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 447707 T828 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 447715 T828 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 447729 T828 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 447762 T828 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 447863 T828 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 447871 T828 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 448179 T828 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 448195 T828 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 448198 T828 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 448210 T828 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 448215 T828 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 448219 T828 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 448220 T828 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 448220 T828 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 448220 T828 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 448221 T828 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 448221 T828 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 448222 T828 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 448222 T828 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 448222 T828 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/cores/collection1/data/
   [junit4]   2> 448223 T828 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@78a8ee09
   [junit4]   2> 448224 T828 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/cores/collection1/data
   [junit4]   2> 448224 T828 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/cores/collection1/data/index/
   [junit4]   2> 448224 T828 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 448225 T828 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/cores/collection1/data/index
   [junit4]   2> 448225 T828 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=18, maxMergeAtOnceExplicit=29, maxMergedSegmentMB=19.7001953125, floorSegmentMB=1.7607421875, forceMergeDeletesPctAllowed=20.3593349700404, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8547633598335697
   [junit4]   2> 448226 T828 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@77eb2a1b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7019901e),segFN=segments_1,generation=1}
   [junit4]   2> 448226 T828 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 448233 T828 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 448233 T828 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 448233 T828 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 448234 T828 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 448234 T828 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 448234 T828 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 448235 T828 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 448235 T828 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 448235 T828 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 448255 T828 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 448257 T828 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 448260 T828 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 448262 T828 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 448268 T828 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/json/docs,/get,/admin/properties,/config,/admin/logging,/admin/threads,standard,/admin/system,/admin/ping,/admin/segments,/admin/plugins,/update/csv,/admin/file,/admin/mbeans,/schema,/update,/admin/luke,/replication,/update/json
   [junit4]   2> 448270 T828 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 448272 T828 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 448272 T828 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 448273 T828 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 448274 T828 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 448275 T828 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1118247169, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 448276 T828 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@77eb2a1b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7019901e),segFN=segments_1,generation=1}
   [junit4]   2> 448276 T828 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 448276 T828 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@7a9e0d6d[collection1] main
   [junit4]   2> 448277 T828 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 448278 T828 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 448278 T828 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 448279 T828 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 448279 T828 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 448279 T828 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 448280 T828 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 448280 T828 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 448280 T828 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 448280 T828 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 448281 T829 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7a9e0d6d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 448282 T828 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 448283 T832 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:51153/p_myp collection:control_collection shard:shard1
   [junit4]   2> 448283 T798 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 448283 T798 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 448283 T832 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 448286 T798 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 448287 T798 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 448288 T832 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 448289 T836 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@15489fd2 name:ZooKeeperConnection Watcher:127.0.0.1:51149/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 448289 T798 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 448290 T824 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 448290 T798 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 448290 T832 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 448291 T832 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 448290 T798 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C269 name=collection1 org.apache.solr.core.SolrCore@743f1f8a url=http://127.0.0.1:51153/p_myp/collection1 node=127.0.0.1:51153_p_myp C269_STATE=coll:control_collection core:collection1 props:{state=down, core=collection1, node_name=127.0.0.1:51153_p_myp, base_url=http://127.0.0.1:51153/p_myp}
   [junit4]   2> 448291 T832 C:control_collection S:shard1 C269 P51153 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:51153/p_myp/collection1/
   [junit4]   2> 448291 T825 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 448292 T832 C:control_collection S:shard1 C269 P51153 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 448292 T832 C:control_collection S:shard1 C269 P51153 oasc.SyncStrategy.syncToMe http://127.0.0.1:51153/p_myp/collection1/ has no replicas
   [junit4]   2> 448293 T832 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:51153/p_myp/collection1/ shard1
   [junit4]   2> 448293 T832 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 448294 T836 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 448294 T824 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 448296 T798 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 448297 T824 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 448298 T825 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51153/p_myp",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 448401 T836 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 448401 T824 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 448450 T832 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:51153/p_myp/collection1/ and leader is http://127.0.0.1:51153/p_myp/collection1/
   [junit4]   2> 448451 T832 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:51153/p_myp
   [junit4]   2> 448451 T832 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 448451 T832 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 448451 T832 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 448452 T824 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 448453 T825 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:51153_p_myp",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51153/p_myp",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null} current state version: 3
   [junit4]   2> 448454 T825 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:51153_p_myp",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51153/p_myp",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 448557 T824 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 448557 T836 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 449531 T798 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/cores/collection1
   [junit4]   2> 449534 T798 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001
   [junit4]   2> 449535 T798 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 449537 T798 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:51164
   [junit4]   2> 449537 T798 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solrconfig=solrconfig.xml, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/tempDir-001/jetty1, hostPort=51163, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/cores, hostContext=/p_myp}
   [junit4]   2> 449537 T798 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 449538 T798 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/'
   [junit4]   2> 449567 T798 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/solr.xml
   [junit4]   2> 449606 T798 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/cores
   [junit4]   2> 449606 T798 oasc.CoreContainer.<init> New CoreContainer 610610784
   [junit4]   2> 449606 T798 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/]
   [junit4]   2> 449606 T798 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/lib
   [junit4]   2> 449607 T798 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/lib).
   [junit4]   2> 449617 T798 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 449617 T798 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 449617 T798 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 449618 T798 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 449618 T798 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 449618 T798 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 449618 T798 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 449619 T798 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 449619 T798 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 449619 T798 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 449619 T798 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 449620 T798 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 449620 T798 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 449620 T798 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 449620 T798 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 449621 T798 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51149/solr
   [junit4]   2> 449621 T798 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 449621 T798 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 449622 T798 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 449623 T848 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@31e57021 name:ZooKeeperConnection Watcher:127.0.0.1:51149 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 449623 T798 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 449624 T798 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 449626 T798 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 449627 T851 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@72932842 name:ZooKeeperConnection Watcher:127.0.0.1:51149/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 449627 T798 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 449631 T798 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 450634 T798 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51163_p_myp
   [junit4]   2> 450635 T798 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51163_p_myp
   [junit4]   2> 450637 T798 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 450640 T798 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/cores
   [junit4]   2> 450642 T798 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/cores/collection1/, name=collection1, config=solrconfig.xml, dataDir=data/, collection=collection1, transient=false, loadOnStartup=true, coreNodeName=, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/cores/collection1, shard=}
   [junit4]   2> 450643 T798 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/cores/collection1/
   [junit4]   2> 450643 T798 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 450645 T852 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 450645 T852 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 450646 T824 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 450646 T852 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 450647 T825 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:51163_p_myp",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51163/p_myp",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null} current state version: 4
   [junit4]   2> 450647 T825 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:51163_p_myp",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51163/p_myp",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 450648 T825 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 450648 T825 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 450649 T851 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 450649 T836 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 450649 T824 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 451646 T852 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 451647 T852 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 451648 T852 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 451648 T852 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 451649 T852 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 451649 T852 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/cores/collection1/'
   [junit4]   2> 451666 T852 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 451669 T852 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 451680 T852 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 451702 T852 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 451752 T852 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 451853 T852 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 451865 T852 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 452307 T852 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 452325 T852 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 452329 T852 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 452340 T852 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 452345 T852 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 452350 T852 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 452351 T852 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 452351 T852 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 452351 T852 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 452352 T852 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 452352 T852 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 452353 T852 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 452353 T852 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 452353 T852 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 452354 T852 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@78a8ee09
   [junit4]   2> 452354 T852 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/cores/collection1/data
   [junit4]   2> 452355 T852 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 452355 T852 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 452355 T852 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 452356 T852 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=18, maxMergeAtOnceExplicit=29, maxMergedSegmentMB=19.7001953125, floorSegmentMB=1.7607421875, forceMergeDeletesPctAllowed=20.3593349700404, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8547633598335697
   [junit4]   2> 452357 T852 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4fd319a7 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2c394be2),segFN=segments_1,generation=1}
   [junit4]   2> 452357 T852 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 452365 T852 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 452365 T852 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 452365 T852 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 452366 T852 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 452366 T852 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 452366 T852 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 452367 T852 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 452367 T852 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 452367 T852 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 452395 T852 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 452398 T852 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 452400 T852 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 452403 T852 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 452420 T852 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/json/docs,/get,/admin/properties,/config,/admin/logging,/admin/threads,standard,/admin/system,/admin/ping,/admin/segments,/admin/plugins,/update/csv,/admin/file,/admin/mbeans,/schema,/update,/admin/luke,/replication,/update/json
   [junit4]   2> 452422 T852 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 452424 T852 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 452425 T852 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 452427 T852 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 452427 T852 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 452428 T852 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1118247169, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 452429 T852 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4fd319a7 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2c394be2),segFN=segments_1,generation=1}
   [junit4]   2> 452429 T852 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 452430 T852 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4c479bf7[collection1] main
   [junit4]   2> 452430 T852 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 452431 T852 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 452431 T852 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 452432 T852 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 452432 T852 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 452432 T852 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 452433 T852 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 452433 T852 C:collection1 c:collection1 

[...truncated too long message...]

r/live_nodes
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 		4	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1529366 T6482 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 10 connections to: http://127.0.0.1:57725/z/su
   [junit4]   2> 1529366 T6482 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:57731/z/su
   [junit4]   2> 1529367 T6482 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 1 connections to: http://127.0.0.1:57715/z/su
   [junit4]   2> 1529367 T6482 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:57741/z/su
   [junit4]   2> 1529368 T6482 C:control_collection S:shard1 c:collection1 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1529368 T6482 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:57725/z/su
   [junit4]   2> 1529369 T6482 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:57731/z/su
   [junit4]   2> 1529369 T6482 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:57715/z/su
   [junit4]   2> 1529369 T6482 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:57741/z/su
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=94BF5CA3001FF936 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=de_AT -Dtests.timezone=America/St_Barthelemy -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   78.4s J0 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:57725/z/su/repfacttest_c8n_1x3_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([94BF5CA3001FF936:1CEB6379AEE394CE]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:57725/z/su/repfacttest_c8n_1x3_shard1_replica1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 53 more
   [junit4]   2> 1529401 T6482 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 94BF5CA3001FF936-001
   [junit4]   2> NOTE: test params are: codec=Lucene50, sim=DefaultSimilarity, locale=de_AT, timezone=America/St_Barthelemy
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=158683704,total=327155712
   [junit4]   2> NOTE: All tests run in this JVM: [TestConfigOverlay, TestEmbeddedSolrServerConstructors, SignatureUpdateProcessorFactoryTest, DistributedIntervalFacetingTest, DefaultValueUpdateProcessorTest, MoreLikeThisHandlerTest, HdfsUnloadDistributedZkTest, TestPostingsSolrHighlighter, TestSolrConfigHandlerConcurrent, AtomicUpdatesTest, TestZkChroot, TestFuzzyAnalyzedSuggestions, TestLFUCache, EchoParamsTest, BlockCacheTest, TermVectorComponentTest, HardAutoCommitTest, TestBinaryResponseWriter, ClusterStateTest, PeerSyncTest, TestJettySolrRunner, ExternalFileFieldSortTest, TestReRankQParserPlugin, IndexBasedSpellCheckerTest, SpatialFilterTest, TestSuggestSpellingConverter, DateFieldTest, TestWordDelimiterFilterFactory, TestCollationFieldDocValues, TestSolrIndexConfig, TestPivotHelperCode, TestIntervalFaceting, TestRTGBase, OverseerStatusTest, SolrIndexSplitterTest, WordBreakSolrSpellCheckerTest, TestSolrDeletionPolicy1, TestSolrCoreProperties, PluginInfoTest, TimeZoneUtilsTest, TestDistribDocBasedVersion, InfoHandlerTest, DistributedQueueTest, ZkStateWriterTest, TestDynamicFieldCollectionResource, TestLMJelinekMercerSimilarityFactory, ShowFileRequestHandlerTest, HdfsRecoveryZkTest, ShardRoutingTest, DistributedFacetPivotWhiteBoxTest, SortByFunctionTest, TestGroupingSearch, TestRangeQuery, TestIndexSearcher, DistributedFacetPivotSmallAdvancedTest, TestMiniSolrCloudCluster, TestTrackingShardHandlerFactory, HttpPartitionTest, ShardSplitTest, TestSolrConfigHandlerCloud, ReplicationFactorTest]
   [junit4] Completed on J0 in 78.39s, 1 test, 1 error <<< FAILURES!

[...truncated 697 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 4 errors, 61 ignored (21 assumptions)

Total time: 52 minutes 44 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 228586426 bytes
Compression is 0.0%
Took 42 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2878 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2878/

4 tests failed.
REGRESSION:  org.apache.solr.handler.component.DistributedMLTComponentTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:50578/r/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:50578/r/collection1
	at __randomizedtesting.SeedInfo.seed([7FD3AC6109AC3620:F78793BBA7505BD8]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:566)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:558)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:606)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:588)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:567)
	at org.apache.solr.handler.component.DistributedMLTComponentTest.test(DistributedMLTComponentTest.java:126)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 51 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:37228/_bog/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37228/_bog/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([7FD3AC6109AC3620:F78793BBA7505BD8]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([7FD3AC6109AC3620:F78793BBA7505BD8]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([7FD3AC6109AC3620:F78793BBA7505BD8]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9257 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/init-core-data-001
   [junit4]   2> 112850 T215 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /ebfhh/
   [junit4]   2> 112866 T215 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 112867 T216 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 112967 T215 oasc.ZkTestServer.run start zk server on port:36921
   [junit4]   2> 112968 T215 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 112970 T215 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 112976 T223 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5375b99f name:ZooKeeperConnection Watcher:127.0.0.1:36921 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 112977 T215 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 112978 T215 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 112978 T215 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 112983 T215 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 112984 T215 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 112987 T226 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@33d40863 name:ZooKeeperConnection Watcher:127.0.0.1:36921/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 112987 T215 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 112989 T215 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 112989 T215 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 112992 T215 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 112995 T215 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 112997 T215 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 113000 T215 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 113001 T215 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 113005 T215 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 113006 T215 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 113010 T215 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 113011 T215 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 113014 T215 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 113015 T215 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 113017 T215 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 113018 T215 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 113021 T215 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 113022 T215 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 113025 T215 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 113026 T215 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 113029 T215 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 113030 T215 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 113032 T215 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 113033 T215 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 113035 T215 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 113036 T215 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 113039 T215 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 113040 T215 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 115068 T215 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/cores/collection1
   [junit4]   2> 115074 T215 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 115080 T215 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:31943
   [junit4]   2> 115081 T215 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/ebfhh, hostPort=16926, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/cores}
   [junit4]   2> 115083 T215 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@5fb271f1
   [junit4]   2> 115084 T215 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/'
   [junit4]   2> 115168 T215 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/solr.xml
   [junit4]   2> 115291 T215 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/cores
   [junit4]   2> 115292 T215 oasc.CoreContainer.<init> New CoreContainer 1738635257
   [junit4]   2> 115293 T215 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/]
   [junit4]   2> 115293 T215 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/lib
   [junit4]   2> 115295 T215 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/lib).
   [junit4]   2> 115314 T215 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 115315 T215 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 115316 T215 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 115316 T215 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 115317 T215 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 115318 T215 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 115318 T215 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 115319 T215 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 115319 T215 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 115320 T215 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 115320 T215 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 115321 T215 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 115321 T215 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 115322 T215 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 115322 T215 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 115323 T215 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:36921/solr
   [junit4]   2> 115324 T215 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 115325 T215 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 115326 T215 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 115329 T238 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3533cc3 name:ZooKeeperConnection Watcher:127.0.0.1:36921 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 115330 T215 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 115331 T215 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 115336 T215 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 115338 T241 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6e692fe name:ZooKeeperConnection Watcher:127.0.0.1:36921/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 115339 T215 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 115342 T215 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 115346 T215 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 115349 T215 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 115352 T215 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 115355 T215 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 115358 T215 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 115361 T215 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 115363 T215 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 115365 T215 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:16926_ebfhh
   [junit4]   2> 115366 T215 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:16926_ebfhh
   [junit4]   2> 115369 T215 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 115371 T215 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 115373 T215 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 115375 T215 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:16926_ebfhh
   [junit4]   2> 115376 T215 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 115379 T215 oasc.Overseer.start Overseer (id=93585283107192835-127.0.0.1:16926_ebfhh-n_0000000000) starting
   [junit4]   2> 115382 T215 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 115391 T215 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 115393 T243 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 115393 T215 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 115394 T242 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 115400 T215 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/cores
   [junit4]   2> 115404 T215 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/cores/collection1, dataDir=data/, shard=, schema=schema.xml, loadOnStartup=true, transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/cores/collection1/, collection=control_collection, config=solrconfig.xml, name=collection1}
   [junit4]   2> 115404 T215 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/cores/collection1/
   [junit4]   2> 115405 T215 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 115407 T245 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 115408 T245 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 115409 T245 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 115409 T241 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 115425 T242 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:16926/ebfhh",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:16926_ebfhh",
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 115426 T242 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:16926/ebfhh",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:16926_ebfhh",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 115429 T242 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 115430 T242 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 115434 T241 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 116410 T245 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 116411 T245 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 116412 T245 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 116412 T245 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 116413 T245 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 116414 T245 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/cores/collection1/'
   [junit4]   2> 116436 T245 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 116439 T245 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 116456 T245 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 116483 T245 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 116555 T245 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 116557 T245 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 116586 T245 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 117147 T245 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 117170 T245 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 117174 T245 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 117200 T245 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 117260 T245 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 117268 T245 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 117269 T245 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 117270 T245 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 117270 T245 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 117272 T245 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 117272 T245 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 117272 T245 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 117273 T245 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 117273 T245 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/cores/collection1/data/
   [junit4]   2> 117274 T245 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4e965d5
   [junit4]   2> 117275 T245 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/cores/collection1/data
   [junit4]   2> 117275 T245 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/cores/collection1/data/index/
   [junit4]   2> 117276 T245 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 117276 T245 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/cores/collection1/data/index
   [junit4]   2> 117277 T245 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=22, maxMergedSegmentMB=84.2001953125, floorSegmentMB=0.2041015625, forceMergeDeletesPctAllowed=22.20860105460578, segmentsPerTier=29.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 117278 T245 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@21ae0eca lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2a4d010c),segFN=segments_1,generation=1}
   [junit4]   2> 117279 T245 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 117289 T245 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 117290 T245 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 117290 T245 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 117291 T245 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 117291 T245 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 117292 T245 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 117292 T245 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 117293 T245 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 117293 T245 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 117316 T245 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 117320 T245 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 117323 T245 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 117326 T245 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 117334 T245 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/luke,standard,/config,/admin/mbeans,/admin/segments,/admin/logging,/get,/admin/file,/admin/plugins,/admin/ping,/admin/system,/schema,/admin/properties,/update/json,/update/json/docs,/replication,/admin/threads,/update/csv,/update
   [junit4]   2> 117338 T245 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 117341 T245 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 117342 T245 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 117344 T245 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 117345 T245 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 117347 T245 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1200085076, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 117348 T245 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@21ae0eca lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2a4d010c),segFN=segments_1,generation=1}
   [junit4]   2> 117349 T245 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 117350 T245 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@35a0c598[collection1] main
   [junit4]   2> 117351 T245 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 117352 T245 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 117353 T245 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 117353 T245 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 117354 T245 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 117354 T245 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 117355 T245 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 117356 T245 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 117356 T245 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 117356 T245 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 117358 T246 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@35a0c598[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 117359 T245 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 117360 T249 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:16926/ebfhh collection:control_collection shard:shard1
   [junit4]   2> 117360 T215 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 117361 T215 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 117361 T249 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 117365 T215 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 117367 T215 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 117369 T253 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4a60166d name:ZooKeeperConnection Watcher:127.0.0.1:36921/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 117369 T249 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 117369 T215 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 117371 T215 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 117371 T215 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 117372 T241 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 117372 T249 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 117373 T249 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 117373 T242 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C60 name=collection1 org.apache.solr.core.SolrCore@34c927df url=http://127.0.0.1:16926/ebfhh/collection1 node=127.0.0.1:16926_ebfhh C60_STATE=coll:control_collection core:collection1 props:{state=down, core=collection1, base_url=http://127.0.0.1:16926/ebfhh, node_name=127.0.0.1:16926_ebfhh}
   [junit4]   2> 117373 T249 C:control_collection S:shard1 C60 P16926 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:16926/ebfhh/collection1/
   [junit4]   2> 117374 T249 C:control_collection S:shard1 C60 P16926 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 117375 T249 C:control_collection S:shard1 C60 P16926 oasc.SyncStrategy.syncToMe http://127.0.0.1:16926/ebfhh/collection1/ has no replicas
   [junit4]   2> 117375 T215 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 117376 T253 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 117376 T241 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 117376 T249 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:16926/ebfhh/collection1/ shard1
   [junit4]   2> 117376 T215 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 117379 T249 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 117379 T215 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 117381 T215 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 117383 T256 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@623c3ef0 name:ZooKeeperConnection Watcher:127.0.0.1:36921/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 117384 T215 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 117385 T215 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 117386 T241 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 117387 T242 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:16926/ebfhh",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 117392 T242 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 117393 T242 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 117394 T253 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 117394 T241 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 117399 T241 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 117400 T242 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 117402 T253 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 117402 T241 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 117437 T249 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:16926/ebfhh/collection1/ and leader is http://127.0.0.1:16926/ebfhh/collection1/
   [junit4]   2> 117438 T249 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:16926/ebfhh
   [junit4]   2> 117439 T249 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 117439 T249 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 117440 T249 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 117442 T241 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 117446 T242 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:16926/ebfhh",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:16926_ebfhh",
   [junit4]   2> 	  "operation":"state"} current state version: 4
   [junit4]   2> 117447 T242 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:16926/ebfhh",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:16926_ebfhh",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 117553 T253 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 117553 T241 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 119030 T215 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/cores/collection1
   [junit4]   2> 119034 T215 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001
   [junit4]   2> 119035 T215 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 119038 T215 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:60867
   [junit4]   2> 119039 T215 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/ebfhh, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/cores, hostPort=52711, solrconfig=solrconfig.xml}
   [junit4]   2> 119040 T215 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@5fb271f1
   [junit4]   2> 119040 T215 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/'
   [junit4]   2> 119098 T215 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/solr.xml
   [junit4]   2> 119164 T215 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/cores
   [junit4]   2> 119165 T215 oasc.CoreContainer.<init> New CoreContainer 383362931
   [junit4]   2> 119165 T215 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/]
   [junit4]   2> 119165 T215 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/lib
   [junit4]   2> 119166 T215 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/lib).
   [junit4]   2> 119181 T215 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 119181 T215 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 119182 T215 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 119183 T215 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 119183 T215 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 119184 T215 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 119184 T215 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 119184 T215 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 119185 T215 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 119185 T215 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 119186 T215 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 119187 T215 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 119187 T215 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 119188 T215 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 119188 T215 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 119188 T215 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:36921/solr
   [junit4]   2> 119189 T215 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 119189 T215 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 119191 T215 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 119198 T268 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7191b8be name:ZooKeeperConnection Watcher:127.0.0.1:36921 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 119199 T215 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 119200 T215 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 119203 T215 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 119205 T271 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@617ceddc name:ZooKeeperConnection Watcher:127.0.0.1:36921/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 119205 T215 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 119211 T215 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 120218 T215 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52711_ebfhh
   [junit4]   2> 120219 T215 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:52711_ebfhh
   [junit4]   2> 120223 T215 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 120226 T215 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/cores
   [junit4]   2> 120230 T215 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/cores/collection1, schema=schema.xml, name=collection1, shard=, transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/cores/collection1/, dataDir=data/, loadOnStartup=true, config=solrconfig.xml, coreNodeName=}
   [junit4]   2> 120230 T215 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/cores/collection1/
   [junit4]   2> 120231 T215 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 120234 T272 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 120235 T272 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 120237 T241 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 120238 T272 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 120238 T272 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 120238 T242 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:52711/ebfhh",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:52711_ebfhh",
   [junit4]   2> 	  "operation":"state"} current state version: 5
   [junit4]   2> 120239 T272 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 120239 T242 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:52711/ebfhh",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:52711_ebfhh",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 120240 T242 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 120240 T242 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 120240 T272 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 120241 T272 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 120243 T242 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 120245 T271 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 120245 T271 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 121241 T272 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 121242 T272 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 121243 T272 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 121243 T272 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 121244 T272 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 121245 T272 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/cores/collection1/'
   [junit4]   2> 121264 T272 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 121267 T272 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 121283 T272 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 121313 T272 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 121380 T272 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 121382 T272 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 121405 T272 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 121938 T272 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 121959 T272 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 121964 T272 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 121989 T272 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 121997 T272 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 122003 T272 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 122004 T272 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 122004 T272 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 122005 T272 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 122006 T272 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 122006 T272 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 122007 T272 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 122007 T272 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 122007 T272 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 122008 T272 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4e965d5
   [junit4]   2> 122010 T272 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/cores/collection1/data
   [junit4]   2> 122010 T272 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 122010 T272 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 122012 T272 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 122012 T272 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=22, maxMergedSegmentMB=84.2001953125, floorSegmentMB=0.2041015625, forceMergeDeletesPctAllowed=22.20860105460578, segmentsPerTier=29.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 122013 T272 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@70817d51 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@454a8a8b),segFN=segments_1,generation=1}
   [junit4]   2> 122014 T272 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 122022 T272 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 122023 T272 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 122023 T272 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 122024 T272 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 122024 T272 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 122024 T272 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 122025 T272 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 122025 T272 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 122025 T272 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 122039 T272 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 122041 T272 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 122043 T272 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 122046 T272 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 122053 T272 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/luke,standard,/config,/admin/mbeans,/admin/segments,/admin/logging,/get,/admin/file,/admin/plugins,/admin/ping,/admin/system,/schema,/admin/properties,/update/json,/update/json/docs,/replication,/admin/threads,/update/csv,/update
   [junit4]   2> 122056 T272 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 122057 T272 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 122057 T272 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 122059 T272 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 122059 T272 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 122061 T272 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1200085076, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 122062 T272 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@70817d51 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4

[...truncated too long message...]

aliases.json
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		4	/solr/collections/collection1/state.json
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent data watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		2	/solr/collections/c8n_1x3_commits/leader_elect/shard1/election/93585371076821001-core_node1-n_0000000000
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/queue
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1532959 T6241 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:37228/_bog
   [junit4]   2> 1532960 T6241 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37254/_bog
   [junit4]   2> 1532960 T6241 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37165/_bog
   [junit4]   2> 1532961 T6241 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37209/_bog
   [junit4]   2> 1532961 T6241 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 11 connections to: http://127.0.0.1:37189/_bog
   [junit4]   2> 1532961 T6241 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37228/_bog
   [junit4]   2> 1532962 T6241 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37254/_bog
   [junit4]   2> 1532962 T6241 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37165/_bog
   [junit4]   2> 1532962 T6241 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37209/_bog
   [junit4]   2> 1532962 T6241 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37189/_bog
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=7FD3AC6109AC3620 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ar_MA -Dtests.timezone=America/Argentina/Rio_Gallegos -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   78.2s J2 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37228/_bog/c8n_1x3_commits_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([7FD3AC6109AC3620:F78793BBA7505BD8]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 48 more
   [junit4]   2> 1532986 T6241 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7FD3AC6109AC3620-001
   [junit4]   2> 78159 T6240 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=ar_MA, timezone=America/Argentina/Rio_Gallegos
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=120600672,total=369623040
   [junit4]   2> NOTE: All tests run in this JVM: [WordBreakSolrSpellCheckerTest, TestDistributedSearch, TestPerFieldSimilarity, TestHighlightDedupGrouping, SharedFSAutoReplicaFailoverTest, TestBadConfig, TestCharFilters, TestWriterPerf, TestSerializedLuceneMatchVersion, TestFieldTypeResource, TestSweetSpotSimilarityFactory, TestDistributedGrouping, TestCollapseQParserPlugin, TestMaxScoreQueryParser, SuggesterFSTTest, TestReRankQParserPlugin, TestLRUStatsCache, DeleteShardTest, TestLazyCores, MinimalSchemaTest, ScriptEngineTest, SolrCmdDistributorTest, TestCloudSchemaless, TestExactSharedStatsCache, AddSchemaFieldsUpdateProcessorFactoryTest, PeerSyncTest, TestCloudPivotFacet, TestFastWriter, TestFieldCollectionResource, SchemaVersionSpecificBehaviorTest, DisMaxRequestHandlerTest, SolrIndexSplitterTest, SpellCheckComponentTest, TestNonDefinedSimilarityFactory, DistributedFacetPivotWhiteBoxTest, TestMergePolicyConfig, IndexSchemaRuntimeFieldTest, TestRestManager, EnumFieldTest, QueryEqualityTest, TestInitQParser, DistribCursorPagingTest, TestExtendedDismaxParser, HdfsWriteToMultipleCollectionsTest, DeleteInactiveReplicaTest, CircularListTest, QueryElevationComponentTest, ZkControllerTest, TestRandomFaceting, TestSolrConfigHandlerCloud, TestUtils, TestBlendedInfixSuggestions, TestSolrDynamicMBean, TestShortCircuitedRequests, SyncSliceTest, TestFiltering, CurrencyFieldXmlFileTest, ShardSplitTest, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J2 in 78.70s, 1 test, 1 error <<< FAILURES!

[...truncated 711 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 4 errors, 61 ignored (21 assumptions)

Total time: 53 minutes 58 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 214333640 bytes
Compression is 0.0%
Took 38 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2877 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2877/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:50644/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50644/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([EAB2019C6240E07D:62E63E46CCBC8D85]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:17642/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:17642/collection1
	at __randomizedtesting.SeedInfo.seed([EAB2019C6240E07D:62E63E46CCBC8D85]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:17642/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:18480/kl_/o/repfacttest_c8n_1x3_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:18480/kl_/o/repfacttest_c8n_1x3_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([EAB2019C6240E07D:62E63E46CCBC8D85]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:18480/kl_/o/repfacttest_c8n_1x3_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9318 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/init-core-data-001
   [junit4]   2> 131839 T341 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 131848 T341 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 131849 T342 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 131949 T341 oasc.ZkTestServer.run start zk server on port:17610
   [junit4]   2> 131953 T341 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 131954 T341 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 131958 T349 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3892a8c3 name:ZooKeeperConnection Watcher:127.0.0.1:17610 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 131958 T341 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 131959 T341 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 131959 T341 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 131964 T341 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 131965 T341 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 131967 T352 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@92e546b name:ZooKeeperConnection Watcher:127.0.0.1:17610/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 131967 T341 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 131968 T341 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 131968 T341 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 131973 T341 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 131977 T341 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 131979 T341 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 131984 T341 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 131984 T341 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 131990 T341 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 131991 T341 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 131996 T341 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 131997 T341 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 132001 T341 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 132002 T341 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 132006 T341 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 132007 T341 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 132011 T341 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 132012 T341 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 132016 T341 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 132016 T341 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 132020 T341 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 132021 T341 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 132025 T341 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 132026 T341 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 132030 T341 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 132030 T341 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 132034 T341 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 132035 T341 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 133733 T341 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/cores/collection1
   [junit4]   2> 133739 T341 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 133749 T341 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:17615
   [junit4]   2> 133749 T341 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/cores, hostPort=17614, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/tempDir-001/control/data}
   [junit4]   2> 133750 T341 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 133751 T341 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/'
   [junit4]   2> 133805 T341 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/solr.xml
   [junit4]   2> 133877 T341 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/cores
   [junit4]   2> 133877 T341 oasc.CoreContainer.<init> New CoreContainer 1810978688
   [junit4]   2> 133878 T341 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/]
   [junit4]   2> 133878 T341 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/lib
   [junit4]   2> 133879 T341 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/lib).
   [junit4]   2> 133894 T341 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 133895 T341 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 133895 T341 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 133895 T341 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 133896 T341 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 133896 T341 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 133897 T341 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 133897 T341 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 133897 T341 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 133898 T341 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 133898 T341 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 133899 T341 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 133900 T341 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 133900 T341 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 133900 T341 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 133901 T341 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:17610/solr
   [junit4]   2> 133901 T341 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 133902 T341 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 133903 T341 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 133905 T364 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@a8a4795 name:ZooKeeperConnection Watcher:127.0.0.1:17610 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 133906 T341 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 133907 T341 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 133910 T341 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 133912 T367 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1b778e41 name:ZooKeeperConnection Watcher:127.0.0.1:17610/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 133912 T341 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 133915 T341 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 133919 T341 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 133924 T341 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 133928 T341 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 133933 T341 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 133938 T341 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 133942 T341 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 133945 T341 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 133948 T341 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:17614_
   [junit4]   2> 133949 T341 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:17614_
   [junit4]   2> 133952 T341 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 133955 T341 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 133958 T341 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 133962 T341 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:17614_
   [junit4]   2> 133962 T341 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 133965 T341 oasc.Overseer.start Overseer (id=93583903181307907-127.0.0.1:17614_-n_0000000000) starting
   [junit4]   2> 133969 T341 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 133982 T341 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 133983 T369 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 133983 T341 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 133984 T368 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 133992 T341 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/cores
   [junit4]   2> 134007 T341 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=control_collection, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/cores/collection1/, dataDir=data/, shard=, config=solrconfig.xml, schema=schema.xml, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/cores/collection1, name=collection1, coreNodeName=, loadOnStartup=true}
   [junit4]   2> 134007 T341 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/cores/collection1/
   [junit4]   2> 134008 T341 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 134010 T371 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 134011 T371 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 134020 T367 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 134020 T371 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 134029 T368 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:17614",
   [junit4]   2> 	  "node_name":"127.0.0.1:17614_"} current state version: 0
   [junit4]   2> 134090 T368 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:17614",
   [junit4]   2> 	  "node_name":"127.0.0.1:17614_"}
   [junit4]   2> 134097 T368 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 134155 T368 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 134159 T367 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 135022 T371 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 135022 T371 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 135023 T371 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 135024 T371 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 135025 T371 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 135026 T371 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/cores/collection1/'
   [junit4]   2> 135089 T371 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 135120 T371 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 135163 T371 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 135198 T371 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 135261 T371 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 135264 T371 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 135283 T371 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 135799 T371 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 135824 T371 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 135828 T371 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 135861 T371 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 135869 T371 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 135876 T371 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 135878 T371 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 135878 T371 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 135879 T371 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 135881 T371 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 135881 T371 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 135882 T371 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 135882 T371 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 135883 T371 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/cores/collection1/data/
   [junit4]   2> 135883 T371 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@77d0bef0
   [junit4]   2> 135884 T371 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/cores/collection1/data
   [junit4]   2> 135885 T371 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/cores/collection1/data/index/
   [junit4]   2> 135885 T371 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 135886 T371 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/cores/collection1/data/index
   [junit4]   2> 135886 T371 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=20, maxMergeAtOnceExplicit=28, maxMergedSegmentMB=64.7998046875, floorSegmentMB=0.791015625, forceMergeDeletesPctAllowed=25.56615195121221, segmentsPerTier=37.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 135887 T371 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@29ee6f9e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@64e20393),segFN=segments_1,generation=1}
   [junit4]   2> 135888 T371 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 135897 T371 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 135987 T371 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 135988 T371 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 135988 T371 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 135988 T371 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 135989 T371 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 135989 T371 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 135990 T371 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 135990 T371 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 136013 T371 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 136016 T371 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 136018 T371 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 136021 T371 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 136028 T371 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: standard,/admin/properties,/admin/segments,/admin/threads,/update/csv,/admin/system,/update,/replication,/admin/plugins,/admin/luke,/schema,/admin/file,/admin/mbeans,/admin/logging,/admin/ping,/config,/get,/update/json/docs,/update/json
   [junit4]   2> 136031 T371 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 136033 T371 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 136033 T371 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 136036 T371 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 136036 T371 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 136037 T371 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=19, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 136038 T371 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@29ee6f9e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@64e20393),segFN=segments_1,generation=1}
   [junit4]   2> 136038 T371 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 136039 T371 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@726aef5c[collection1] main
   [junit4]   2> 136039 T371 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 136041 T371 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 136041 T371 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 136056 T371 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 136056 T371 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 136057 T371 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 136058 T371 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 136058 T371 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 136059 T371 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 136059 T371 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 136060 T372 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@726aef5c[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 136068 T371 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 136070 T375 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:17614 collection:control_collection shard:shard1
   [junit4]   2> 136070 T341 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 136071 T341 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 136086 T341 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 136087 T341 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 136090 T379 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7940bbc5 name:ZooKeeperConnection Watcher:127.0.0.1:17610/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 136090 T341 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 136092 T341 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 136093 T341 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 136102 T341 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 136103 T341 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 136103 T341 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 136104 T341 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 136106 T382 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@521f3cfb name:ZooKeeperConnection Watcher:127.0.0.1:17610/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 136107 T341 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 136107 T341 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 136109 T375 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 136111 T367 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 136114 T368 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 1
   [junit4]   2> 136114 T368 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 136118 T368 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 136120 T375 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 136121 T367 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 136121 T379 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 136125 T375 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 136125 T375 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 136130 T368 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 2
   [junit4]   2> ASYNC  NEW_CORE C25 name=collection1 org.apache.solr.core.SolrCore@321ec652 url=http://127.0.0.1:17614/collection1 node=127.0.0.1:17614_ C25_STATE=coll:control_collection core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:17614, node_name=127.0.0.1:17614_}
   [junit4]   2> 136139 T375 C:control_collection S:shard1 C25 P17614 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:17614/collection1/
   [junit4]   2> 136140 T375 C:control_collection S:shard1 C25 P17614 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 136140 T375 C:control_collection S:shard1 C25 P17614 oasc.SyncStrategy.syncToMe http://127.0.0.1:17614/collection1/ has no replicas
   [junit4]   2> 136140 T375 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:17614/collection1/ shard1
   [junit4]   2> 136150 T367 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 136153 T379 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 136153 T367 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 136157 T375 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 136167 T367 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 136169 T368 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:17614",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 3
   [junit4]   2> 136276 T379 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 136276 T367 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 136322 T375 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:17614/collection1/ and leader is http://127.0.0.1:17614/collection1/
   [junit4]   2> 136322 T375 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:17614
   [junit4]   2> 136322 T375 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 136323 T375 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 136323 T375 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 136326 T367 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 136328 T368 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:17614",
   [junit4]   2> 	  "node_name":"127.0.0.1:17614_"} current state version: 4
   [junit4]   2> 136330 T368 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:17614",
   [junit4]   2> 	  "node_name":"127.0.0.1:17614_"}
   [junit4]   2> 136437 T367 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 136437 T379 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 137799 T341 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/cores/collection1
   [junit4]   2> 137803 T341 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001
   [junit4]   2> 137804 T341 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 137807 T341 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:17643
   [junit4]   2> 137808 T341 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/cores, hostContext=/, hostPort=17642, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/tempDir-001/jetty1}
   [junit4]   2> 137809 T341 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 137809 T341 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/'
   [junit4]   2> 137864 T341 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/solr.xml
   [junit4]   2> 137931 T341 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/cores
   [junit4]   2> 137932 T341 oasc.CoreContainer.<init> New CoreContainer 1978325
   [junit4]   2> 137933 T341 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/]
   [junit4]   2> 137933 T341 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/lib
   [junit4]   2> 137934 T341 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/lib).
   [junit4]   2> 137950 T341 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 137951 T341 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 137951 T341 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 137952 T341 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 137952 T341 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 137952 T341 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 137953 T341 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 137953 T341 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 137954 T341 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 137954 T341 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 137954 T341 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 137955 T341 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 137956 T341 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 137956 T341 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 137957 T341 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 137957 T341 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:17610/solr
   [junit4]   2> 137957 T341 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 137958 T341 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 137959 T341 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 137961 T394 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4c0af79c name:ZooKeeperConnection Watcher:127.0.0.1:17610 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 137962 T341 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 137963 T341 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 137966 T341 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 137969 T397 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2a87393d name:ZooKeeperConnection Watcher:127.0.0.1:17610/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 137969 T341 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 137979 T341 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 138991 T341 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:17642_
   [junit4]   2> 138992 T341 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:17642_
   [junit4]   2> 138997 T341 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 139002 T341 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/cores
   [junit4]   2> 139005 T341 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {loadOnStartup=true, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/cores/collection1, config=solrconfig.xml, shard=, coreNodeName=, dataDir=data/, transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/cores/collection1/, collection=collection1, schema=schema.xml, name=collection1}
   [junit4]   2> 139005 T341 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/cores/collection1/
   [junit4]   2> 139006 T341 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 139010 T398 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 139010 T398 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 139013 T367 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 139015 T398 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 139015 T368 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:17642",
   [junit4]   2> 	  "node_name":"127.0.0.1:17642_"} current state version: 5
   [junit4]   2> 139015 T398 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 139017 T368 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:17642",
   [junit4]   2> 	  "node_name":"127.0.0.1:17642_"}
   [junit4]   2> 139017 T398 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 139017 T368 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 139018 T368 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 139021 T368 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 139029 T398 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 139030 T398 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 139030 T398 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 139030 T398 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 139031 T398 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 139031 T398 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 139033 T398 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 139033 T398 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/cores/collection1/'
   [junit4]   2> 139052 T398 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 139056 T398 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 139071 T398 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 139096 T398 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 139155 T398 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 139157 T398 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 139177 T398 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 139676 T398 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 139698 T398 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 139701 T398 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 139730 T398 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 139736 T398 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 139742 T398 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 139744 T398 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 139744 T398 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 139745 T398 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 139746 T398 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 139746 T398 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 139747 T398 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 139747 T398 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 139748 T398 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 139748 T398 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@77d0bef0
   [junit4]   2> 139749 T398 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/cores/collection1/data
   [junit4]   2> 139749 T398 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 139750 T398 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 139750 T398 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 139751 T398 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=20, maxMergeAtOnceExplicit=28, maxMergedSegmentMB=64.7998046875, floorSegmentMB=0.791015625, forceMergeDeletesPctAllowed=25.56615195121221, segmentsPerTier=37.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 139752 T398 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1edb0c6d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3f0cd7ee),segFN=segments_1,generation=1}
   [junit4]   2> 139752 T398 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 139763 T398 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 139764 T398 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 139764 T398 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 139765 T398 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 139765 T398 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 139766 T398 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 139766 T398 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 139767 T398 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 139767 T398 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 139788 T398 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 139791 T398 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 139794 T398 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 139796 T398 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 139837 T398 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: standard,/admin/properties,/admin/segments,/admin/threads,/update/csv,/admin/system,/update,/replication,/admin/plugins,/admin/luke,/schema,/admin/file,/admin/mbeans,/admin/logging,/admin/ping,/config,/get,/update/json/docs,/update/json
   [junit4]   2> 139840 T398 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 139842 T398 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 139842 T398 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 139844 T398 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 139844 T398 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 139845 T398 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=19, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 139846 T398 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1edb0c6d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3f0cd7ee),segFN=segments_1,generation=1}
   [junit4]   2> 139847 T398 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 139847 T398 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@5aba3e7f[collection1] main
   [junit4]   2> 139847 T398 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 139849 T398 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 139850 T398 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 139850 T398 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 139851 T398 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 139851 T398 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 139852 T398 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 139852 T398 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 139852 T398 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 139853 T398 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 139854 T399 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5aba3e7f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 139854 T398 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 139855 T402 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:17642 collection:collection1 shard:shard1
   [junit4]   2> 139856 T341 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 139857 T341 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 139857 T402 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 139864 T402 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 139867 T367 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 139867 T402 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 139868 T402 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C26 name=collection1 org.apache.solr.core.SolrCore@46980bbf url=http://127.0.0.1:17642/collection1 node=127.0.0.1:17642_ C26_STATE=coll:collection1 core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:17642, node_name=127.0.0.1:17642_}
   [junit4]   2> 139868 T402 C:collection1 S:shard1 C26 P17642 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:17642/collection1/
   [junit4]   2> 139868 T402 C:collection1 S:shard1 C26 P17642 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 139868 T368 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 139868 T402 C:collection1 S:shard1 C26 P17642 oasc.SyncStrategy.syncToMe http://127.0.0.1:17642/collection1/ has no replicas
   [junit4]   2> 139869 T402 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:17642/collection1/ shard1
   [junit4]   2> 139869 T402 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 139871 T368 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 139873 T397 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 139874 T397 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 139878 T367 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 139879 T368 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:17642",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 139882 T368 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 139883 T397 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 139884 T397 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 139930 T402 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:17642/collection1/ and leader is http://127.0.0.1:17642/collection1/
   [junit4]   2> 139931 T402 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:17642
   [junit4]   2> 139931 T402 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 139931 T402 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 139932 T402 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 139934 T367 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 139936 T368 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = 

[...truncated too long message...]

e.json
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		4	/solr/collections/collection1/state.json
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent data watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		2	/solr/collections/c8n_1x3_commits/leader_elect/shard1/election/93583987909263373-core_node3-n_0000000000
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1503904 T7084 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:50662/
   [junit4]   2> 1503904 T7084 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:50644/
   [junit4]   2> 1503904 T7084 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:40631/
   [junit4]   2> 1503904 T7084 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 10 connections to: http://127.0.0.1:50635/
   [junit4]   2> 1503904 T7084 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 3 connections to: http://127.0.0.1:50625/
   [junit4]   2> 1503905 T7084 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:50662/
   [junit4]   2> 1503905 T7084 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:50644/
   [junit4]   2> 1503905 T7084 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:40631/
   [junit4]   2> 1503905 T7084 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:50635/
   [junit4]   2> 1503906 T7084 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:50625/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=EAB2019C6240E07D -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=hi_IN -Dtests.timezone=Australia/Eucla -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   79.7s J2 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50644/c8n_1x3_commits_shard1_replica2
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([EAB2019C6240E07D:62E63E46CCBC8D85]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 48 more
   [junit4]   2> 1503926 T7084 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest EAB2019C6240E07D-001
   [junit4]   2> 79740 T7083 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=DefaultSimilarity, locale=hi_IN, timezone=Australia/Eucla
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=151235736,total=389545984
   [junit4]   2> NOTE: All tests run in this JVM: [TestCryptoKeys, ScriptEngineTest, SolrTestCaseJ4Test, ShardSplitTest, OpenExchangeRatesOrgProviderTest, TestMiniSolrCloudCluster, DocValuesMissingTest, CurrencyFieldOpenExchangeTest, DocValuesTest, TestJmxMonitoredMap, DistribDocExpirationUpdateProcessorTest, TestShardHandlerFactory, TestBulkSchemaConcurrent, DocExpirationUpdateProcessorFactoryTest, CachingDirectoryFactoryTest, StatelessScriptUpdateProcessorFactoryTest, DistanceFunctionTest, TestReplicaProperties, TestFileDictionaryLookup, SolrXmlInZkTest, TestExactStatsCache, SegmentsInfoRequestHandlerTest, TestQuerySenderNoQuery, TestLRUCache, RequestHandlersTest, DirectSolrConnectionTest, CoreMergeIndexesAdminHandlerTest, TestClusterStateMutator, TestInfoStreamLogging, TestIBSimilarityFactory, DistributedQueueTest, ReturnFieldsTest, CSVRequestHandlerTest, SimpleFacetsTest, TestRestManager, ClusterStateUpdateTest, TestXIncludeConfig, TestLeaderElectionZkExpiry, RAMDirectoryFactoryTest, TestUniqueKeyFieldResource, CircularListTest, FacetPivotSmallTest, TestReload, BasicZkTest, TestPerFieldSimilarity, TestRebalanceLeaders, OverseerCollectionProcessorTest, TestDistributedSearch, DefaultValueUpdateProcessorTest, TestReRankQParserPlugin, TestCustomSort, AlternateDirectoryTest, PrimUtilsTest, TestSolrDynamicMBean, SchemaVersionSpecificBehaviorTest, ExternalCollectionsTest, SolrIndexSplitterTest, ClusterStateTest, TestCloudSchemaless, TestCloudManagedSchemaConcurrent, ChaosMonkeyNothingIsSafeTest, UnloadDistributedZkTest, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J2 in 80.02s, 1 test, 1 error <<< FAILURES!

[...truncated 714 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 53 minutes 1 second
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 239090085 bytes
Compression is 0.0%
Took 44 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2876 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2876/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:24846/hff/p/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:24846/hff/p/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([177722754B01BEA:89234DFDFA4C7612]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([177722754B01BEA:89234DFDFA4C7612]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([177722754B01BEA:89234DFDFA4C7612]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9763 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/init-core-data-001
   [junit4]   2> 749957 T1332 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /_ww/ab
   [junit4]   2> 749969 T1332 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 749970 T1333 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 750070 T1332 oasc.ZkTestServer.run start zk server on port:25128
   [junit4]   2> 750071 T1332 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 750072 T1332 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 750076 T1340 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5001b139 name:ZooKeeperConnection Watcher:127.0.0.1:25128 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 750077 T1332 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 750077 T1332 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 750078 T1332 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 750080 T1332 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 750081 T1332 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 750083 T1343 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3ec5b62d name:ZooKeeperConnection Watcher:127.0.0.1:25128/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 750084 T1332 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 750084 T1332 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 750085 T1332 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 750087 T1332 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 750089 T1332 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 750091 T1332 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 750093 T1332 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 750094 T1332 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 750096 T1332 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 750097 T1332 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 750099 T1332 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 750100 T1332 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 750102 T1332 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 750102 T1332 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 750104 T1332 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 750105 T1332 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 750107 T1332 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 750107 T1332 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 750110 T1332 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 750110 T1332 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 750112 T1332 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 750112 T1332 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 750114 T1332 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 750115 T1332 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 750116 T1332 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 750117 T1332 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 750119 T1332 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 750119 T1332 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 751484 T1332 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/cores/collection1
   [junit4]   2> 751488 T1332 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 751494 T1332 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:25136
   [junit4]   2> 751494 T1332 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=25135, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/cores, hostContext=/_ww/ab, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/tempDir-001/control/data}
   [junit4]   2> 751495 T1332 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 751496 T1332 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/'
   [junit4]   2> 751541 T1332 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/solr.xml
   [junit4]   2> 751588 T1332 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/cores
   [junit4]   2> 751589 T1332 oasc.CoreContainer.<init> New CoreContainer 544904992
   [junit4]   2> 751589 T1332 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/]
   [junit4]   2> 751589 T1332 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/lib
   [junit4]   2> 751590 T1332 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/lib).
   [junit4]   2> 751605 T1332 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 751606 T1332 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 751606 T1332 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 751606 T1332 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 751606 T1332 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 751607 T1332 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 751607 T1332 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 751607 T1332 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 751607 T1332 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 751608 T1332 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 751608 T1332 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 751608 T1332 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 751609 T1332 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 751609 T1332 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 751609 T1332 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 751609 T1332 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:25128/solr
   [junit4]   2> 751610 T1332 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 751610 T1332 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 751611 T1332 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 751613 T1355 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@68cbe092 name:ZooKeeperConnection Watcher:127.0.0.1:25128 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 751614 T1332 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 751614 T1332 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 751617 T1332 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 751619 T1358 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1e8b171f name:ZooKeeperConnection Watcher:127.0.0.1:25128/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 751619 T1332 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 751621 T1332 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 751623 T1332 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 751625 T1332 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 751626 T1332 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 751628 T1332 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 751630 T1332 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 751631 T1332 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 751632 T1332 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 751634 T1332 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:25135__ww%2Fab
   [junit4]   2> 751634 T1332 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:25135__ww%2Fab
   [junit4]   2> 751636 T1332 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 751637 T1332 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 751638 T1332 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 751640 T1332 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:25135__ww%2Fab
   [junit4]   2> 751640 T1332 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 751641 T1332 oasc.Overseer.start Overseer (id=93583003170504707-127.0.0.1:25135__ww%2Fab-n_0000000000) starting
   [junit4]   2> 751643 T1332 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 751648 T1332 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 751649 T1360 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 751649 T1332 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 751650 T1359 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 751654 T1332 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/cores
   [junit4]   2> 751656 T1332 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=control_collection, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/cores/collection1/, name=collection1, config=solrconfig.xml, dataDir=data/, schema=schema.xml, loadOnStartup=true, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/cores/collection1, shard=, transient=false}
   [junit4]   2> 751656 T1332 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/cores/collection1/
   [junit4]   2> 751656 T1332 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 751657 T1362 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 751658 T1362 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 751659 T1362 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 751659 T1358 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 751660 T1359 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25135/_ww/ab",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:25135__ww%2Fab",
   [junit4]   2> 	  "roles":null} current state version: 0
   [junit4]   2> 751660 T1359 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25135/_ww/ab",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:25135__ww%2Fab",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 751660 T1359 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 751661 T1359 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 751662 T1358 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 752660 T1362 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 752660 T1362 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 752661 T1362 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 752661 T1362 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 752662 T1362 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 752662 T1362 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/cores/collection1/'
   [junit4]   2> 752687 T1362 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 752690 T1362 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 752709 T1362 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 752762 T1362 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 752818 T1362 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 752819 T1362 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 752830 T1362 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 753286 T1362 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 753308 T1362 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 753312 T1362 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 753328 T1362 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 753334 T1362 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 753341 T1362 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 753343 T1362 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 753343 T1362 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 753344 T1362 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 753344 T1362 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 753344 T1362 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 753345 T1362 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 753345 T1362 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 753345 T1362 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/cores/collection1/data/
   [junit4]   2> 753345 T1362 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@29e4c4ed
   [junit4]   2> 753346 T1362 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/cores/collection1/data
   [junit4]   2> 753347 T1362 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/cores/collection1/data/index/
   [junit4]   2> 753347 T1362 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 753347 T1362 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/cores/collection1/data/index
   [junit4]   2> 753347 T1362 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=27, maxMergedSegmentMB=53.33203125, floorSegmentMB=1.701171875, forceMergeDeletesPctAllowed=13.492737200540025, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1879079618145032
   [junit4]   2> 753348 T1362 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=RawDirectoryWrapper(RAMDirectory@4bd9b020 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@40ba2bc7),segFN=segments_1,generation=1}
   [junit4]   2> 753348 T1362 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 753354 T1362 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 753354 T1362 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 753355 T1362 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 753355 T1362 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 753355 T1362 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 753356 T1362 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 753356 T1362 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 753357 T1362 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 753357 T1362 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 753381 T1362 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 753386 T1362 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 753390 T1362 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 753393 T1362 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 753403 T1362 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/logging,/config,/get,/admin/properties,/admin/luke,/update/json,/admin/segments,/update,/admin/mbeans,/replication,/admin/file,/schema,/admin/ping,/update/json/docs,standard,/admin/plugins,/admin/system,/update/csv,/admin/threads
   [junit4]   2> 753406 T1362 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 753408 T1362 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 753408 T1362 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 753410 T1362 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 753410 T1362 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 753411 T1362 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=10, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 753412 T1362 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=RawDirectoryWrapper(RAMDirectory@4bd9b020 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@40ba2bc7),segFN=segments_1,generation=1}
   [junit4]   2> 753412 T1362 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 753412 T1362 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@38ae1e39[collection1] main
   [junit4]   2> 753413 T1362 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 753414 T1362 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 753414 T1362 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 753414 T1362 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 753415 T1362 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 753415 T1362 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 753415 T1362 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 753416 T1362 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 753416 T1362 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 753416 T1362 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 753419 T1363 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@38ae1e39[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 753420 T1362 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 753421 T1366 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:25135/_ww/ab collection:control_collection shard:shard1
   [junit4]   2> 753421 T1332 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 753421 T1332 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 753422 T1366 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 753423 T1332 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 753424 T1332 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 753427 T1370 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@d4e80da name:ZooKeeperConnection Watcher:127.0.0.1:25128/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 753427 T1332 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 753428 T1332 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 753428 T1332 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 753428 T1366 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 753432 T1358 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 753432 T1366 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 753433 T1366 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 753433 T1359 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C248 name=collection1 org.apache.solr.core.SolrCore@1aa95cd7 url=http://127.0.0.1:25135/_ww/ab/collection1 node=127.0.0.1:25135__ww%2Fab C248_STATE=coll:control_collection core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:25135/_ww/ab, node_name=127.0.0.1:25135__ww%2Fab}
   [junit4]   2> 753433 T1366 C:control_collection S:shard1 C248 P25135 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:25135/_ww/ab/collection1/
   [junit4]   2> 753433 T1332 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 753434 T1366 C:control_collection S:shard1 C248 P25135 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 753434 T1332 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 753435 T1366 C:control_collection S:shard1 C248 P25135 oasc.SyncStrategy.syncToMe http://127.0.0.1:25135/_ww/ab/collection1/ has no replicas
   [junit4]   2> 753435 T1358 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 753435 T1370 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 753435 T1332 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 753435 T1366 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:25135/_ww/ab/collection1/ shard1
   [junit4]   2> 753437 T1366 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 753437 T1332 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 753441 T1373 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1070bc26 name:ZooKeeperConnection Watcher:127.0.0.1:25128/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 753441 T1332 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 753441 T1332 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 753444 T1358 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 753446 T1359 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 2, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 753447 T1359 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 753450 T1359 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25135/_ww/ab",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 753451 T1359 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 753452 T1370 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 753452 T1358 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 753456 T1358 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 753457 T1370 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 753457 T1358 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 753498 T1366 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:25135/_ww/ab/collection1/ and leader is http://127.0.0.1:25135/_ww/ab/collection1/
   [junit4]   2> 753498 T1366 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:25135/_ww/ab
   [junit4]   2> 753499 T1366 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 753499 T1366 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 753499 T1366 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 753501 T1358 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 753502 T1359 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25135/_ww/ab",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:25135__ww%2Fab",
   [junit4]   2> 	  "roles":null} current state version: 4
   [junit4]   2> 753503 T1359 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25135/_ww/ab",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:25135__ww%2Fab",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 753607 T1358 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 753607 T1370 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 755088 T1332 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/cores/collection1
   [junit4]   2> 755090 T1332 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001
   [junit4]   2> 755091 T1332 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 755094 T1332 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:25162
   [junit4]   2> 755095 T1332 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/cores, hostPort=25161, solrconfig=solrconfig.xml, hostContext=/_ww/ab, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/tempDir-001/jetty1}
   [junit4]   2> 755095 T1332 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 755096 T1332 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/'
   [junit4]   2> 755132 T1332 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/solr.xml
   [junit4]   2> 755173 T1332 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/cores
   [junit4]   2> 755174 T1332 oasc.CoreContainer.<init> New CoreContainer 2110856931
   [junit4]   2> 755174 T1332 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/]
   [junit4]   2> 755174 T1332 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/lib
   [junit4]   2> 755175 T1332 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/lib).
   [junit4]   2> 755188 T1332 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 755188 T1332 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 755188 T1332 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 755189 T1332 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 755189 T1332 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 755189 T1332 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 755189 T1332 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 755190 T1332 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 755190 T1332 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 755190 T1332 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 755190 T1332 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 755191 T1332 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 755191 T1332 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 755191 T1332 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 755192 T1332 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 755192 T1332 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:25128/solr
   [junit4]   2> 755192 T1332 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 755192 T1332 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 755193 T1332 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 755195 T1385 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@21164eff name:ZooKeeperConnection Watcher:127.0.0.1:25128 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 755196 T1332 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 755197 T1332 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 755199 T1332 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 755200 T1388 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7daa8efb name:ZooKeeperConnection Watcher:127.0.0.1:25128/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 755200 T1332 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 755204 T1332 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 756208 T1332 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:25161__ww%2Fab
   [junit4]   2> 756209 T1332 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:25161__ww%2Fab
   [junit4]   2> 756212 T1332 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 756215 T1332 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/cores
   [junit4]   2> 756217 T1332 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, schema=schema.xml, shard=, collection=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/cores/collection1, config=solrconfig.xml, transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/cores/collection1/, dataDir=data/, coreNodeName=, loadOnStartup=true}
   [junit4]   2> 756217 T1332 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/cores/collection1/
   [junit4]   2> 756217 T1332 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 756219 T1389 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 756219 T1389 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 756221 T1358 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 756221 T1389 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 756221 T1389 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 756221 T1359 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25161/_ww/ab",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:25161__ww%2Fab",
   [junit4]   2> 	  "roles":null} current state version: 5
   [junit4]   2> 756222 T1389 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 756222 T1359 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25161/_ww/ab",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:25161__ww%2Fab",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 756222 T1359 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 756222 T1359 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 756223 T1389 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 756223 T1389 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 756224 T1359 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 756224 T1388 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 756225 T1388 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 757224 T1389 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 757224 T1389 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 757225 T1389 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 757225 T1389 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 757225 T1389 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 757226 T1389 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/cores/collection1/'
   [junit4]   2> 757249 T1389 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 757253 T1389 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 757265 T1389 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 757286 T1389 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 757330 T1389 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 757332 T1389 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 757341 T1389 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 757858 T1389 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 757880 T1389 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 757884 T1389 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 757899 T1389 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 757905 T1389 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 757911 T1389 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 757911 T1389 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 757912 T1389 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 757912 T1389 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 757913 T1389 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 757913 T1389 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 757913 T1389 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 757914 T1389 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 757914 T1389 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 757914 T1389 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@29e4c4ed
   [junit4]   2> 757915 T1389 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/cores/collection1/data
   [junit4]   2> 757916 T1389 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 757916 T1389 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 757917 T1389 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 757917 T1389 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=27, maxMergedSegmentMB=53.33203125, floorSegmentMB=1.701171875, forceMergeDeletesPctAllowed=13.492737200540025, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1879079618145032
   [junit4]   2> 757918 T1389 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=RawDirectoryWrapper(RAMDirectory@3c50c7d0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5b6e479f),segFN=segments_1,generation=1}
   [junit4]   2> 757918 T1389 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 757927 T1389 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 757927 T1389 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 757928 T1389 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 757928 T1389 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 757928 T1389 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 757928 T1389 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 757929 T1389 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 757930 T1389 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 757930 T1389 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 757950 T1389 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 757954 T1389 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 757958 T1389 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 757961 T1389 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 757980 T1389 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/logging,/config,/get,/admin/properties,/admin/luke,/update/json,/admin/segments,/update,/admin/mbeans,/replication,/admin/file,/schema,/admin/ping,/update/json/docs,standard,/admin/plugins,/admin/system,/update/csv,/admin/threads
   [junit4]   2> 757982 T1389 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 757983 T1389 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 757983 T1389 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 757985 T1389 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 757985 T1389 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 757986 T1389 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=10, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 757987 T1389 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=RawDirectoryWrapper(RAMDirectory@3c50c7d0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5b6e479f),segFN=segments_1,generation=1}
   [junit4]   2> 757987 T1389 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 757987 T1389 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@34ba55b1[collection1] main
   [junit4]   2> 757988 T1389 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 757988 T1389 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 757988 T1389 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 757989 T1389 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 757989 T1389 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 757989 T1389 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 757990 T1389 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 757990 T1389 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 757990 T1389 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 757990 T1389 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 757992 T1390 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@34ba55b1[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 757992 T1389 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 757993 T1393 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:25161/_ww/ab collection:collection1 shard:shard1
   [junit4]   2> 757993 T1332 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 757994 T1332 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 757994 T1393 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 757997 T1393 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 757999 T1358 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 757999 T1393 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 757999 T1393 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C249 name=collection1 org.apache.solr.core.SolrCore@28eef1bd url=http://127.0.0.1:25161/_ww/ab/collection1 node=127.0.0.1:25161__ww%2Fab C249_STATE=coll:collection1 core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:25161/_ww/ab, node_name=127.0.0.1:25161__ww%2Fab}
   [junit4]   2> 757999 T1393 C:collection1 S:shard1 C249 P25161 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:25161/_ww/ab/collection1/
   [junit4]   2> 758000 T1359 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> ASYNC  NEW_CORE C250 name=collection1 org.apache.solr.core.SolrCore@28eef1bd url=http://127.0.0.1:25161/_ww/ab/collection1 node=127.0.0.1:25161__ww%2Fab C250_STATE=coll:collection1 core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:25161/_ww/ab, node_name=127.0.0.1:25161__ww%2Fab}
   [junit4]   2> 758011 T1393 C:collection1 S:shard1 C250 P25161 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 758011 T1393 C:collection1 S:shard1 C250 P25161 oasc.SyncStrategy.syncToMe http://127.0.0.1:25161/_ww/ab/collection1/ has no replicas
   [junit4]   2> 758012 T1393 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:25161/_ww/ab/collection1/ shard1
   [junit4]   2> 758012 T1393 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 758012 T1359 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 758013 T1388 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 758014 T1388 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 758015 T1359 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25161/_ww/ab",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 758016 T1359 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 758017 T1388 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 758018 T1388 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 758019 T1358 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 758066 T1393 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:25161/_ww/ab/collection1/ and leader is http://127.0.0.1:25161/_ww/ab/collection1/
   [junit4]   2> 758067 T1393 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:25161/_ww/ab
   [junit4]   2> 758067 T1393 C:collection1 S:shard1 oasc.ZkController.checkRecovery

[...truncated too long message...]

22776 T7433 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1522779 T7322 C:control_collection S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 1522961 T7322 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:24981 24981
   [junit4]   2> 1523046 T7323 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:24981 24981
   [junit4]   2> 1523047 T7323 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 		2	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1523047 T7322 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:25001/
   [junit4]   2> 1523047 T7322 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:25007/
   [junit4]   2> 1523047 T7322 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:24989/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=177722754B01BEA -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=sr_CS -Dtests.timezone=Pacific/Enderbury -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   70.0s J2 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([177722754B01BEA:89234DFDFA4C7612]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1523069 T7322 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 177722754B01BEA-001
   [junit4]   2> 69981 T7321 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50), sim=DefaultSimilarity, locale=sr_CS, timezone=Pacific/Enderbury
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=182519552,total=326631424
   [junit4]   2> NOTE: All tests run in this JVM: [HighlighterMaxOffsetTest, TestRawResponseWriter, XsltUpdateRequestHandlerTest, TestFieldResource, TestStressReorder, TestRecoveryHdfs, AtomicUpdatesTest, OpenCloseCoreStressTest, TestReload, SchemaVersionSpecificBehaviorTest, DocValuesMultiTest, UniqFieldsUpdateProcessorFactoryTest, SpellPossibilityIteratorTest, SyncSliceTest, TestDistribDocBasedVersion, ShowFileRequestHandlerTest, TestRemoteStreaming, TestPivotHelperCode, InfoHandlerTest, TestTolerantSearch, SegmentsInfoRequestHandlerTest, SolrIndexConfigTest, TestReplicationHandler, DistribDocExpirationUpdateProcessorTest, TestArbitraryIndexDir, TestSolrDeletionPolicy2, SpellCheckComponentTest, TestFuzzyAnalyzedSuggestions, HdfsWriteToMultipleCollectionsTest, TestCollectionAPI, ZkNodePropsTest, TestSweetSpotSimilarityFactory, TestPerFieldSimilarity, TestFiltering, TestRequestStatusCollectionAPI, TestRecovery, TestManagedResource, EnumFieldTest, DebugComponentTest, TestDynamicFieldResource, HdfsDirectoryFactoryTest, TestClassNameShortening, BadCopyFieldTest, TestReplicaProperties, TestSolr4Spatial2, IndexSchemaTest, TestDFRSimilarityFactory, NumericFieldsTest, TestReRankQParserPlugin, ActionThrottleTest, SolrCoreTest, TestStressLucene, TestCodecSupport, TimeZoneUtilsTest, TestJsonFacets, CollectionsAPIDistributedZkTest, VMParamsZkACLAndCredentialsProvidersTest, CacheHeaderTest, TestDistributedSearch, BasicDistributedZkTest, TestSolrConfigHandlerCloud, TestConfigReload, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J2 in 70.76s, 1 test, 1 error <<< FAILURES!

[...truncated 700 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 55 minutes 44 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 236053084 bytes
Compression is 0.0%
Took 45 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2875 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2875/

4 tests failed.
REGRESSION:  org.apache.solr.search.facet.TestJsonFacets.testComplex

Error Message:
mismatch: 'civic'!='a' @ facets/makes/buckets/[0]/models/buckets/[1]/val

Stack Trace:
java.lang.RuntimeException: mismatch: 'civic'!='a' @ facets/makes/buckets/[0]/models/buckets/[1]/val
	at __randomizedtesting.SeedInfo.seed([7739CFC0B2BE137A:96E6CA5C9EF05F19]:0)
	at org.apache.solr.SolrTestCaseHS.matchJSON(SolrTestCaseHS.java:160)
	at org.apache.solr.SolrTestCaseHS.assertJQ(SolrTestCaseHS.java:142)
	at org.apache.solr.SolrTestCaseHS$Client.testJQ(SolrTestCaseHS.java:288)
	at org.apache.solr.search.facet.TestJsonFacets.testComplex(TestJsonFacets.java:155)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:51177/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:51177/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([7739CFC0B2BE137A:FF6DF01A1C427E82]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([7739CFC0B2BE137A:FF6DF01A1C427E82]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([7739CFC0B2BE137A:FF6DF01A1C427E82]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9329 lines...]
   [junit4] Suite: org.apache.solr.search.facet.TestJsonFacets
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.search.facet.TestJsonFacets 7739CFC0B2BE137A-001/init-core-data-001
   [junit4]   2> 229125 T547 oas.SolrTestCaseJ4.initCore ####initCore
   [junit4]   2> 229125 T547 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 229127 T547 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 229128 T547 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 229128 T547 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 229248 T547 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 229273 T547 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 229328 T547 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig-tlog.xml
   [junit4]   2> 229329 T547 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml
   [junit4]   2> 229337 T547 oass.IndexSchema.readSchema [null] Schema name=example
   [junit4]   2> 229575 T547 oass.AbstractSpatialFieldType.init WARN units parameter is deprecated, please use distanceUnits instead for field types with class SpatialRecursivePrefixTreeFieldType
   [junit4]   2> 229582 T547 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 229604 T547 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 229608 T547 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 229612 T547 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 229612 T547 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr
   [junit4]   2> 229612 T547 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/'
   [junit4]   2> 229651 T547 oasc.CoreContainer.<init> New CoreContainer 1879689010
   [junit4]   2> 229651 T547 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/]
   [junit4]   2> 229651 T547 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/lib
   [junit4]   2> 229652 T547 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/lib).
   [junit4]   2> 229662 T547 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 600000
   [junit4]   2> 229662 T547 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 229662 T547 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 60000
   [junit4]   2> 229662 T547 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 229663 T547 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 229663 T547 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 229663 T547 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 229663 T547 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 229663 T547 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 229664 T547 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 229664 T547 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 229664 T547 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=30000&connTimeout=30000&retry=true
   [junit4]   2> 229665 T547 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 229665 T547 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 229665 T547 oasc.CoreContainer.load Node Name: testNode
   [junit4]   2> 229667 T547 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {loadOnStartup=true, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.search.facet.TestJsonFacets 7739CFC0B2BE137A-001/init-core-data-001, collection=collection1, name=collection1, shard=shard1, instanceDir=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, transient=false, config=solrconfig-tlog.xml, schema=schema_latest.xml}
   [junit4]   2> 229667 T548 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 229669 T548 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 229670 T548 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 229670 T548 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 229757 T548 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 229782 T548 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 229823 T548 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig-tlog.xml
   [junit4]   2> 229824 T548 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml
   [junit4]   2> 229832 T548 oass.IndexSchema.readSchema [collection1] Schema name=example
   [junit4]   2> 230022 T548 oass.AbstractSpatialFieldType.init WARN units parameter is deprecated, please use distanceUnits instead for field types with class SpatialRecursivePrefixTreeFieldType
   [junit4]   2> 230030 T548 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 230046 T548 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 230051 T548 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 230056 T548 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 230056 T548 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 230056 T548 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.search.facet.TestJsonFacets 7739CFC0B2BE137A-001/init-core-data-001/
   [junit4]   2> 230057 T548 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5828ba35
   [junit4]   2> 230058 T548 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.search.facet.TestJsonFacets 7739CFC0B2BE137A-001/init-core-data-001
   [junit4]   2> 230058 T548 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.search.facet.TestJsonFacets 7739CFC0B2BE137A-001/init-core-data-001/index/
   [junit4]   2> 230058 T548 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.search.facet.TestJsonFacets 7739CFC0B2BE137A-001/init-core-data-001/index' doesn't exist. Creating new index...
   [junit4]   2> 230059 T548 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.search.facet.TestJsonFacets 7739CFC0B2BE137A-001/init-core-data-001/index
   [junit4]   2> 230059 T548 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=27, maxMergeAtOnceExplicit=18, maxMergedSegmentMB=16.2978515625, floorSegmentMB=0.353515625, forceMergeDeletesPctAllowed=16.295204050591646, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 230060 T548 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6bb394b8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ae8f345),segFN=segments_1,generation=1}
   [junit4]   2> 230061 T548 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 230068 T548 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 230068 T548 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 230069 T548 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 230069 T548 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 230070 T548 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 230070 T548 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 230070 T548 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 230071 T548 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 230071 T548 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 230101 T548 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 230104 T548 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 230107 T548 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 230109 T548 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 230116 T548 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/system,/admin/luke,/schema,standard,/admin/threads,/update/json,/admin/logging,/admin/properties,/admin/plugins,/update/json/docs,/admin/file,/get,/admin/mbeans,/update/csv,/admin/ping,/config,/update,/replication,/admin/segments
   [junit4]   2> 230118 T548 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 230119 T548 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 230120 T548 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 230121 T548 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 230121 T548 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 230122 T548 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=18, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=88.712890625, floorSegmentMB=1.6796875, forceMergeDeletesPctAllowed=20.970532846719955, segmentsPerTier=38.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.37418944491677153
   [junit4]   2> 230122 T548 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6bb394b8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ae8f345),segFN=segments_1,generation=1}
   [junit4]   2> 230123 T548 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 230123 T548 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@51983697[collection1] main
   [junit4]   2> 230124 T548 c:collection1 oasr.ManagedResourceStorage.newStorageIO WARN Cannot write to config directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf; switching to use InMemory storage instead.
   [junit4]   2> 230125 T548 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 230125 T548 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 230125 T548 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 230125 T548 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 230126 T548 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 230127 T549 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@51983697[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 230129 T547 oas.SolrTestCaseJ4.initCore ####initCore end
   [junit4]   2> 230136 T547 oas.SolrTestCaseJ4.setUp ###Starting testStats
   [junit4]   2> ASYNC  NEW_CORE C84 name=collection1 org.apache.solr.core.SolrCore@37d51fe9
   [junit4]   2> 230140 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {deleteByQuery=*:* (-1497312353721516032)} 0 3
   [junit4]   2> 230147 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={versions=true&wt=json&indent=true} {add=[1 (1497312353729904640)]} 0 2
   [junit4]   2> 230149 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={versions=true&wt=json&indent=true} {add=[2 (1497312353733050368)]} 0 1
   [junit4]   2> 230150 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={versions=true&wt=json&indent=true} {add=[3 (1497312353735147520)]} 0 0
   [junit4]   2> 230151 T547 C84 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 230176 T547 C84 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6bb394b8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ae8f345),segFN=segments_1,generation=1}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6bb394b8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ae8f345),segFN=segments_2,generation=2}
   [junit4]   2> 230177 T547 C84 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 230186 T547 C84 oass.SolrIndexSearcher.<init> Opening Searcher@4b7fa047[collection1] main
   [junit4]   2> 230186 T547 C84 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 230187 T549 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4b7fa047[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):c3)))}
   [junit4]   2> 230188 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 38
   [junit4]   2> 230194 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={versions=true&wt=json&indent=true} {add=[4 (1497312353777090560)]} 0 4
   [junit4]   2> 230195 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={versions=true&wt=json&indent=true} {add=[5 (1497312353781284864)]} 0 1
   [junit4]   2> 230196 T547 C84 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 230210 T547 C84 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6bb394b8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ae8f345),segFN=segments_2,generation=2}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6bb394b8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ae8f345),segFN=segments_3,generation=3}
   [junit4]   2> 230211 T547 C84 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
   [junit4]   2> 230212 T547 C84 oass.SolrIndexSearcher.<init> Opening Searcher@18d90a23[collection1] main
   [junit4]   2> 230213 T547 C84 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 230214 T549 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@18d90a23[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):c3) Uninverting(_1(5.2.0):c2)))}
   [junit4]   2> 230215 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 19
   [junit4]   2> 230220 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={versions=true&wt=json&indent=true} {add=[6 (1497312353805402112)]} 0 3
   [junit4]   2> 230221 T547 C84 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 230228 T547 C84 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6bb394b8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ae8f345),segFN=segments_3,generation=3}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6bb394b8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ae8f345),segFN=segments_4,generation=4}
   [junit4]   2> 230228 T547 C84 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 4
   [junit4]   2> 230230 T547 C84 oass.SolrIndexSearcher.<init> Opening Searcher@57dc4a9d[collection1] main
   [junit4]   2> 230230 T547 C84 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 230231 T549 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@57dc4a9d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):c3) Uninverting(_1(5.2.0):c2) Uninverting(_2(5.2.0):c1)))}
   [junit4]   2> 230232 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 12
   [junit4]   2> 230342 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7BcatA%3A%7Bquery%3A%7Bq%3A%27%24%7Bcat_s%7D%3AA%27%7D%7D%2C++catA2%3A%7Bquery%3A%7Bquery%3A%27%24%7Bcat_s%7D%3AA%27%7D%7D%2C++catA3%3A%7Bquery%3A%27%24%7Bcat_s%7D%3AA%27%7D++++%7D&wt=json&indent=true} hits=6 status=0 QTime=108 
   [junit4]   2> 230349 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7B+catB%3A%7Bquery%3A%7Bq%3A%27%24%7Bcat_s%7D%3AB%27%2C+facet%3A%7Bnj%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANJ%27%7D%2C+ny%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANY%27%7D%7D+%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=4 
   [junit4]   2> 230353 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=id%3A%282+3%29&json.facet=%7B+catB%3A%7Bquery%3A%7Bq%3A%27%24%7Bcat_s%7D%3AB%27%2C+facet%3A%7Bnj%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANJ%27%7D%2C+ny%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANY%27%7D%7D+%7D%7D%7D&wt=json&indent=true} hits=2 status=0 QTime=4 
   [junit4]   2> 230357 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7B+catB%3A%7Bquery%3A%7Bq%3A%27%24%7Bcat_s%7D%3AB%27%2C+facet%3A%7Bnj%3A%7Bquery%3A%7Bq%3A%27%24%7Bwhere_s%7D%3ANJ%27%7D%7D%2C+ny%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANY%27%7D%7D+%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=3 
   [junit4]   2> 230542 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bc1%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%7D%2C+c2%3A%7Bfield%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%7D%7D%2C+c3%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%7D%7D++%7D&wt=json&indent=true} hits=6 status=0 QTime=184 
   [junit4]   2> 230544 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+mincount%3A3%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 230546 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=id%3A1&json.facet=%7Bf1%3A%7Bterms%3A%27%24%7Bcat_s%7D%27%7D%7D&wt=json&indent=true} hits=1 status=0 QTime=2 
   [junit4]   2> 230547 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=id%3A1&json.facet=%7BprocessEmpty%3Atrue%2C+f1%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+mincount%3A0%7D%7D%7D&wt=json&indent=true} hits=1 status=0 QTime=1 
   [junit4]   2> 230702 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=id%3A1&json.facet=%7BprocessEmpty%3Atrue%2C+f1%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+mincount%3A0%2C+allBuckets%3Atrue%2C+facet%3A%7Bn1%3A%27sum%28%24%7Bnum_d%7D%29%27%7D++%7D%7D%7D&wt=json&indent=true} hits=1 status=0 QTime=154 
   [junit4]   2> 230705 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27n1+desc%27%2C+facet%3A%7Bn1%3A%27sum%28%24%7Bnum_d%7D%29%27%7D++%7D%7D+%2C+f2%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27n1+asc%27%2C+facet%3A%7Bn1%3A%27sum%28%24%7Bnum_d%7D%29%27%7D++%7D%7D+%7D&wt=json&indent=true} hits=6 status=0 QTime=2 
   [junit4]   2> 230793 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27n1+desc%27%2C+facet%3A%7Bn1%3A%27percentile%28%24%7Bnum_d%7D%2C50%29%27%7D++%7D%7D+%2C+f2%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27n1+asc%27%2C+facet%3A%7Bn1%3A%27percentile%28%24%7Bnum_d%7D%2C50%29%27%7D++%7D%7D+%7D&wt=json&indent=true} hits=6 status=0 QTime=88 
   [junit4]   2> 230796 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27n1+desc%27%2C+facet%3A%7Bn1%3A%27percentile%28%24%7Bnum_d%7D%2C50%2C0%2C100%29%27%7D++%7D%7D+%2C+f2%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27n1+asc%27%2C+facet%3A%7Bn1%3A%27percentile%28%24%7Bnum_d%7D%2C50%2C0%2C100%29%27%7D++%7D%7D+%7D&wt=json&indent=true} hits=6 status=0 QTime=2 
   [junit4]   2> 230799 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27count+desc%27+%7D++%7D+++++++++++%2C+f2%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27count+asc%27++%7D++%7D+++++++++++%2C+f3%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27index+asc%27++%7D++%7D+++++++++++%2C+f4%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27index+desc%27+%7D++%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=2 
   [junit4]   2> 230802 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=id%3A1+id%3A6&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27count+desc%27+%7D++%7D+++++++++++%2C+f2%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27count+asc%27++%7D++%7D%7D&wt=json&indent=true} hits=2 status=0 QTime=2 
   [junit4]   2> 230806 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bcat%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+facet%3A%7Bnj%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANJ%27%7D%7D++++%7D+++%7D%7D+%7D&wt=json&indent=true} hits=6 status=0 QTime=3 
   [junit4]   2> 230811 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=id%3A%282+5+4%29&json.facet=%7Bcat%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+facet%3A%7Bnj%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANJ%27%7D%7D++++%7D+++%7D%7D+%7D&wt=json&indent=true} hits=3 status=0 QTime=4 
   [junit4]   2> 230814 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bsuper_s%7D%2C+prefix%3As%2C+mincount%3A0+%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=2 
   [junit4]   2> 230815 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bsuper_s%7D%2C+prefix%3Attt%2C+mincount%3A0+%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 230816 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bsuper_s%7D%2C+prefix%3Aaaaaaa%2C+mincount%3A0+%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=0 
   [junit4]   2> 230818 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bsuper_s%7D%2C+prefix%3Azzzzzz%2C+mincount%3A0+%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 230819 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bnoexist%7D%2C+missing%3Atrue%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 230820 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bsparse_s%7D%2C+missing%3Atrue+%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 230822 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bsparse_s%7D%2C+missing%3Atrue%2C+facet%3A%7Bx%3A%27sum%28num_d%29%27%7D+++%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 230823 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bsparse_s%7D%2C+missing%3Atrue%2C+prefix%3Aon%2C+facet%3A%7Bx%3A%27sum%28num_d%29%27%7D+++%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=0 
   [junit4]   2> 230836 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bsparse_s%7D%2C+missing%3Atrue%2C+prefix%3Appp%2C+facet%3A%7Bx%3A%27sum%28num_d%29%27%7D+++%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=12 
   [junit4]   2> 230838 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&facet=true&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bcat_s%7D%2C+numBuckets%3Atrue%2C+limit%3A1%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 230839 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&facet=true&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bcat_s%7D%2C+numBuckets%3Atrue%2C+prefix%3AB%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=0 
   [junit4]   2> 230841 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&facet=true&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bcat_s%7D%2C+numBuckets%3Atrue%2C+mincount%3A3%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 230944 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf%3A%7Brange%3A%7Bfield%3A%24%7Bnum_d%7D%2C+start%3A-5%2C+end%3A10%2C+gap%3A5%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=102 
   [junit4]   2> 230948 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf%3A%7Brange%3A%7Bfield%3A%24%7Bnum_d%7D%2C+start%3A-5%2C+end%3A10%2C+gap%3A5%2C+include%3Aupper%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=2 
   [junit4]   2> 231029 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf%3A%7Brange%3A%7Bfield%3A%24%7Bnum_d%7D%2C+start%3A-5%2C+end%3A10%2C+gap%3A5%2C+++facet%3A%7B+x%3A%27sum%28%24%7Bnum_i%7D%29%27%2C+ny%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANY%27%7D%7D+++%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=80 
   [junit4]   2> 231038 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf%3A%7Brange%3A%7Bfield%3A%24%7Bnum_d%7D%2C+start%3A-5%2C+end%3A10%2C+gap%3A5%2C+other%3Aall%2C+++facet%3A%7B+x%3A%27sum%28%24%7Bnum_i%7D%29%27%2C+ny%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANY%27%7D%7D+++%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=7 
   [junit4]   2> 231046 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=id%3A%283+4+6%29&json.facet=%7Bf%3A%7Brange%3A%7Bfield%3A%24%7Bnum_d%7D%2C+start%3A-5%2C+end%3A10%2C+gap%3A5%2C+other%3Aall%2C+++facet%3A%7B+x%3A%27sum%28%24%7Bnum_i%7D%29%27%2C+ny%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANY%27%7D%7D+++%7D%7D%7D&wt=json&indent=true} hits=3 status=0 QTime=7 
   [junit4]   2> 231164 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7B+sum1%3A%27sum%28%24%7Bnum_d%7D%29%27%2C+sumsq1%3A%27sumsq%28%24%7Bnum_d%7D%29%27%2C+avg1%3A%27avg%28%24%7Bnum_d%7D%29%27%2C+min1%3A%27min%28%24%7Bnum_d%7D%29%27%2C+max1%3A%27max%28%24%7Bnum_d%7D%29%27%2C+numwhere%3A%27unique%28%24%7Bwhere_s%7D%29%27%2C+med%3A%27percentile%28%24%7Bnum_d%7D%2C50%29%27%2C+perc%3A%27percentile%28%24%7Bnum_d%7D%2C0%2C50.0%2C100%29%27+%7D&wt=json&indent=true} hits=6 status=0 QTime=117 
   [junit4]   2> 231167 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=id%3ADOESNOTEXIST&json.facet=%7B+sum1%3A%27sum%28%24%7Bnum_d%7D%29%27%2C+sumsq1%3A%27sumsq%28%24%7Bnum_d%7D%29%27%2C+avg1%3A%27avg%28%24%7Bnum_d%7D%29%27%2C+min1%3A%27min%28%24%7Bnum_d%7D%29%27%2C+max1%3A%27max%28%24%7Bnum_d%7D%29%27%2C+numwhere%3A%27unique%28%24%7Bwhere_s%7D%29%27%2C+med%3A%27percentile%28%24%7Bnum_d%7D%2C50%29%27%2C+perc%3A%27percentile%28%24%7Bnum_d%7D%2C0%2C50.0%2C100%29%27+%7D&wt=json&indent=true} hits=0 status=0 QTime=2 
   [junit4]   2> 231221 T547 C84 oassf.UnInvertedField.<init> UnInverted multi-valued field {field=multi_ss,memSize=4515,tindexSize=41,time=20,phase1=20,nTerms=2,bigTerms=2,termInstances=0,uses=0}
   [junit4]   2> 231226 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bcat%3A%7Bterms%3A%7Bfield%3A%27%24%7Bmulti_ss%7D%27%2C+facet%3A%7Bnj%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANJ%27%7D%7D++++%7D+++%7D%7D+%7D&wt=json&indent=true} hits=6 status=0 QTime=59 
   [junit4]   2> 231240 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bx%3A%27unique%28%24%7Bmulti_ss%7D%29%27%2C+y%3A%7Bquery%3A%7Bq%3A%27id%3A2%27%2C+facet%3A%7Bx%3A%27unique%28%24%7Bmulti_ss%7D%29%27%7D+%7D%7D+++%7D&wt=json&indent=true} hits=6 status=0 QTime=14 
   [junit4]   2> 231243 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bx%3A%7Bterms%3A%7Bfield%3A%27%24%7Bmulti_ss%7D%27%2CallBuckets%3Atrue%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=2 
   [junit4]   2> 231255 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&facet=true&facet.version=2&facet.field=%7B%21key%3Df1%7D%24%7Bcat_s%7D&facet.mincount=3&wt=json&indent=true} hits=6 status=0 QTime=12 
   [junit4]   2> 231257 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&facet=true&facet.version=2&facet.field=%7B%21key%3Df1%7D%24%7Bsuper_s%7D&facet.prefix=s&facet.mincount=0&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 231273 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&facet=true&facet.version=2&facet.range=%7B%21key%3Df%7D%24%7Bnum_d%7D&facet.range.start=-5&facet.range.end=10&facet.range.gap=5&f.f.facet.stat=x%3Asum%28%24%7Bnum_i%7D%29&subfacet.f.query=%7B%21key%3Dny%7D%24%7Bwhere_s%7D%3ANY&wt=json&indent=true} hits=6 status=0 QTime=15 
   [junit4]   2> 231276 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&facet=true&facet.version=2&facet.field=%7B%21key%3Df1%7D%24%7Bcat_s%7D&facet.field=%7B%21key%3Df2%7D%24%7Bcat_s%7D&f.f1.facet.sort=n1+desc&f.f1.facet.sort=n1+asc&facet.stat=n1%3Asum%28num_d%29&wt=json&indent=true} hits=6 status=0 QTime=2 
   [junit4]   2> 231287 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=id%3A%283+4+6%29&facet=true&facet.version=2&facet.range=%7B%21key%3Df%7D%24%7Bnum_d%7D&facet.range.start=-5&facet.range.end=10&facet.range.gap=5&f.f.facet.stat=x%3Asum%28%24%7Bnum_i%7D%29&subfacet.f.query=%7B%21key%3Dny%7D%24%7Bwhere_s%7D%3ANY&facet.range.other=all&wt=json&indent=true} hits=3 status=0 QTime=10 
   [junit4]   2> 231293 T547 C84 oass.SolrIndexSearcher.<init> Opening Searcher@6c63f619[collection1] realtime
   [junit4]   2> 231293 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {deleteByQuery=*:* (-1497312354928427008)} 0 5
   [junit4]   2> 231303 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={versions=true&wt=json&indent=true} {add=[1 (1497312354942058496)]} 0 2
   [junit4]   2> 231304 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={versions=true&wt=json&indent=true} {add=[2 (1497312354944155648)]} 0 1
   [junit4]   2> 231305 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={versions=true&wt=json&indent=true} {add=[3 (1497312354946252800)]} 0 1
   [junit4]   2> 231305 T547 C84 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 231333 T547 C84 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6bb394b8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ae8f345),segFN=segments_4,generation=4}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6bb394b8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ae8f345),segFN=segments_5,generation=5}
   [junit4]   2> 231334 T547 C84 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 5
   [junit4]   2> 231349 T547 C84 oass.SolrIndexSearcher.<init> Opening Searcher@7313cfa2[collection1] main
   [junit4]   2> 231350 T547 C84 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 231351 T549 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7313cfa2[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_3(5.2.0):c3)))}
   [junit4]   2> 231352 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 47
   [junit4]   2> 231358 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={versions=true&wt=json&indent=true} {add=[4 (1497312354997633024)]} 0 4
   [junit4]   2> 231359 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={versions=true&wt=json&indent=true} {add=[5 (1497312355001827328)]} 0 1
   [junit4]   2> 231360 T547 C84 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 231368 T547 C84 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6bb394b8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ae8f345),segFN=segments_5,generation=5}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6bb394b8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ae8f345),segFN=segments_6,generation=6}
   [junit4]   2> 231368 T547 C84 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 6
   [junit4]   2> 231370 T547 C84 oass.SolrIndexSearcher.<init> Opening Searcher@4cb88286[collection1] main
   [junit4]   2> 231370 T547 C84 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 231371 T549 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4cb88286[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_3(5.2.0):c3) Uninverting(_4(5.2.0):c2)))}
   [junit4]   2> 231372 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 13
   [junit4]   2> 231377 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={versions=true&wt=json&indent=true} {add=[6 (1497312355018604544)]} 0 3
   [junit4]   2> 231378 T547 C84 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 231387 T547 C84 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6bb394b8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ae8f345),segFN=segments_6,generation=6}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6bb394b8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ae8f345),segFN=segments_7,generation=7}
   [junit4]   2> 231388 T547 C84 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 7
   [junit4]   2> 231390 T547 C84 oass.SolrIndexSearcher.<init> Opening Searcher@2bd27391[collection1] main
   [junit4]   2> 231391 T547 C84 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 231391 T549 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2bd27391[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_3(5.2.0):c3) Uninverting(_4(5.2.0):c2) Uninverting(_5(5.2.0):c1)))}
   [junit4]   2> 231392 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 14
   [junit4]   2> 231414 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7BcatA%3A%7Bquery%3A%7Bq%3A%27%24%7Bcat_s%7D%3AA%27%7D%7D%2C++catA2%3A%7Bquery%3A%7Bquery%3A%27%24%7Bcat_s%7D%3AA%27%7D%7D%2C++catA3%3A%7Bquery%3A%27%24%7Bcat_s%7D%3AA%27%7D++++%7D&wt=json&indent=true} hits=6 status=0 QTime=20 
   [junit4]   2> 231417 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7B+catB%3A%7Bquery%3A%7Bq%3A%27%24%7Bcat_s%7D%3AB%27%2C+facet%3A%7Bnj%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANJ%27%7D%2C+ny%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANY%27%7D%7D+%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=3 
   [junit4]   2> 231422 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=id%3A%282+3%29&json.facet=%7B+catB%3A%7Bquery%3A%7Bq%3A%27%24%7Bcat_s%7D%3AB%27%2C+facet%3A%7Bnj%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANJ%27%7D%2C+ny%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANY%27%7D%7D+%7D%7D%7D&wt=json&indent=true} hits=2 status=0 QTime=4 
   [junit4]   2> 231425 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7B+catB%3A%7Bquery%3A%7Bq%3A%27%24%7Bcat_s%7D%3AB%27%2C+facet%3A%7Bnj%3A%7Bquery%3A%7Bq%3A%27%24%7Bwhere_s%7D%3ANJ%27%7D%7D%2C+ny%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANY%27%7D%7D+%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=2 
   [junit4]   2> 231427 T547 C84 oassf.UnInvertedField.<init> UnInverted multi-valued field {field=cat_ss,memSize=4446,tindexSize=41,time=0,phase1=0,nTerms=2,bigTerms=1,termInstances=2,uses=0}
   [junit4]   2> 231430 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bc1%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%7D%2C+c2%3A%7Bfield%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%7D%7D%2C+c3%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%7D%7D++%7D&wt=json&indent=true} hits=6 status=0 QTime=4 
   [junit4]   2> 231432 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+mincount%3A3%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 231434 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=id%3A1&json.facet=%7Bf1%3A%7Bterms%3A%27%24%7Bcat_s%7D%27%7D%7D&wt=json&indent=true} hits=1 status=0 QTime=1 
   [junit4]   2> 231437 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=id%3A1&json.facet=%7BprocessEmpty%3Atrue%2C+f1%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+mincount%3A0%7D%7D%7D&wt=json&indent=true} hits=1 status=0 QTime=2 
   [junit4]   2> 231448 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=id%3A1&json.facet=%7BprocessEmpty%3Atrue%2C+f1%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+mincount%3A0%2C+allBuckets%3Atrue%2C+facet%3A%7Bn1%3A%27sum%28%24%7Bnum_d%7D%29%27%7D++%7D%7D%7D&wt=json&indent=true} hits=1 status=0 QTime=11 
   [junit4]   2> 231451 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27n1+desc%27%2C+facet%3A%7Bn1%3A%27sum%28%24%7Bnum_d%7D%29%27%7D++%7D%7D+%2C+f2%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27n1+asc%27%2C+facet%3A%7Bn1%3A%27sum%28%24%7Bnum_d%7D%29%27%7D++%7D%7D+%7D&wt=json&indent=true} hits=6 status=0 QTime=2 
   [junit4]   2> 231454 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27n1+desc%27%2C+facet%3A%7Bn1%3A%27percentile%28%24%7Bnum_d%7D%2C50%29%27%7D++%7D%7D+%2C+f2%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27n1+asc%27%2C+facet%3A%7Bn1%3A%27percentile%28%24%7Bnum_d%7D%2C50%29%27%7D++%7D%7D+%7D&wt=json&indent=true} hits=6 status=0 QTime=2 
   [junit4]   2> 231457 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27n1+desc%27%2C+facet%3A%7Bn1%3A%27percentile%28%24%7Bnum_d%7D%2C50%2C0%2C100%29%27%7D++%7D%7D+%2C+f2%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27n1+asc%27%2C+facet%3A%7Bn1%3A%27percentile%28%24%7Bnum_d%7D%2C50%2C0%2C100%29%27%7D++%7D%7D+%7D&wt=json&indent=true} hits=6 status=0 QTime=2 
   [junit4]   2> 231463 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27count+desc%27+%7D++%7D+++++++++++%2C+f2%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27count+asc%27++%7D++%7D+++++++++++%2C+f3%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27index+asc%27++%7D++%7D+++++++++++%2C+f4%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27index+desc%27+%7D++%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=5 
   [junit4]   2> 231467 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=id%3A1+id%3A6&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27count+desc%27+%7D++%7D+++++++++++%2C+f2%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27count+asc%27++%7D++%7D%7D&wt=json&indent=true} hits=2 status=0 QTime=3 
   [junit4]   2> 231471 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bcat%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+facet%3A%7Bnj%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANJ%27%7D%7D++++%7D+++%7D%7D+%7D&wt=json&indent=true} hits=6 status=0 QTime=3 
   [junit4]   2> 231477 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=id%3A%282+5+4%29&json.facet=%7Bcat%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+facet%3A%7Bnj%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANJ%27%7D%7D++++%7D+++%7D%7D+%7D&wt=json&indent=true} hits=3 status=0 QTime=5 
   [junit4]   2> 231479 T547 C84 oassf.UnInvertedField.<init> UnInverted multi-valued field {field=super_ss,memSize=4358,tindexSize=46,time=1,phase1=1,nTerms=5,bigTerms=0,termInstances=5,uses=0}
   [junit4]   2> 231479 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bsuper_s%7D%2C+prefix%3As%2C+mincount%3A0+%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 231481 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bsuper_s%7D%2C+prefix%3Attt%2C+mincount%3A0+%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 231482 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bsuper_s%7D%2C+prefix%3Aaaaaaa%2C+mincount%3A0+%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 231483 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bsuper_s%7D%2C+prefix%3Azzzzzz%2C+mincount%3A0+%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=0 
   [junit4]   2> 231484 T547 C84 oassf.UnInvertedField.<init> UnInverted multi-valued field {field=noexist_ss,memSize=4288,tindexSize=0,time=0,phase1=0,nTerms=0,bigTerms=0,termInstances=0,uses=0}
   [junit4]   2> 231485 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bnoexist%7D%2C+missing%3Atrue%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 231487 T547 C84 oassf.UnInvertedField.<init> UnInverted multi-valued field {field=sparse_ss,memSize=4355,tindexSize=43,time=1,phase1=1,nTerms=2,bigTerms=0,termInstances=2,uses=0}
   [junit4]   2> 231487 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bsparse_s%7D%2C+missing%3Atrue+%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 231489 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bsparse_s%7D%2C+missing%3Atrue%2C+facet%3A%7Bx%3A%27sum%28num_d%29%27%7D+++%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 231491 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bsparse_s%7D%2C+missing%3Atrue%2C+prefix%3Aon%2C+facet%3A%7Bx%3A%27sum%28num_d%29%27%7D+++%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 231493 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bsparse_s%7D%2C+missing%3Atrue%2C+prefix%3Appp%2C+facet%3A%7Bx%3A%27sum%28num_d%29%27%7D+++%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 231495 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&facet=true&rows=0&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bcat_s%7D%2C+numBuckets%3Atrue%2C+limit%3A1%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=2 
   [junit4]   2> 231496 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&facet=true&rows=0&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bcat_s%7D%2C+numBuckets%3Atrue%2C+prefix%3AB%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 231498 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&facet=true&rows=0&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bcat_s%7D%2C+numBuckets%3Atrue%2C+mincount%3A3%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 231501 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf%3A%7Brange%3A%7Bfield%3A%24%7Bnum_d%7D%2C+start%3A-5%2C+end%3A10%2C+gap%3A5%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=2 
   [junit4]   2> 231504 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf%3A%7Brange%3A%7Bfield%3A%24%7Bnum_d%7D%2C+start%3A-5%2C+end%3A10%2C+gap%3A5%2C+include%3Aupper%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=2 
   [junit4]   2> 231508 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf%3A%7Brange%3A%7Bfield%3A%24%7Bnum_d%7D%2C+start%3A-5%2C+end%3A10%2C+gap%3A5%2C+++facet%3A%7B+x%3A%27sum%28%24%7Bnum_i%7D%29%27%2C+ny%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANY%27%7D%7D+++%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=3 
   [junit4]   2> 231515 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf%3A%7Brange%3A%7Bfield%3A%24%7Bnum_d%7D%2C+start%3A-5%2C+end%3A10%2C+gap%3A5%2C+other%3Aall%2C+++facet%3A%7B+x%3A%27sum%28%24%7Bnum_i%7D%29%27%2C+ny%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANY%27%7D%7D+++%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=6 
   [junit4]   2> 231522 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=id%3A%283+4+6%29&json.facet=%7Bf%3A%7Brange%3A%7Bfield%3A%24%7Bnum_d%7D%2C+start%3A-5%2C+end%3A10%2C+gap%3A5%2C+other%3Aall%2C+++facet%3A%7B+x%3A%27sum%28%24%7Bnum_i%7D%29%27%2C+ny%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANY%27%7D%7D+++%7D%7D%7D&wt=json&indent=true} hits=3 status=0 QTime=6 
   [junit4]   2> 231524 T547 C84 oassf.UnInvertedField.<init> UnInverted multi-valued field {field=where_ss,memSize=4448,tindexSize=42,time=0,phase1=0,nTerms=2,bigTerms=1,termInstances=2,uses=0}
   [junit4]   2> 231526 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7B+sum1%3A%27sum%28%24%7Bnum_d%7D%29%27%2C+sumsq1%3A%27sumsq%28%24%7Bnum_d%7D%29%27%2C+avg1%3A%27avg%28%24%7Bnum_d%7D%29%27%2C+min1%3A%27min%28%24%7Bnum_d%7D%29%27%2C+max1%3A%27max%28%24%7Bnum_d%7D%29%27%2C+numwhere%3A%27unique%28%24%7Bwhere_s%7D%29%27%2C+med%3A%27percentile%28%24%7Bnum_d%7D%2C50%29%27%2C+perc%3A%27percentile%28%24%7Bnum_d%7D%2C0%2C50.0%2C100%29%27+%7D&wt=json&indent=true} hits=6 status=0 QTime=3 
   [junit4]   2> 231528 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss

[...truncated too long message...]


   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		4	/solr/collections/collection1/state.json
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent data watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		2	/solr/collections/c8n_1x3_commits/leader_elect/shard1/election/93582112433373197-core_node3-n_0000000000
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1689972 T5544 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:51201/
   [junit4]   2> 1689972 T5544 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:51161/
   [junit4]   2> 1689972 T5544 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 3 connections to: http://127.0.0.1:51185/
   [junit4]   2> 1689973 T5544 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:51215/
   [junit4]   2> 1689973 T5544 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 13 connections to: http://127.0.0.1:51177/
   [junit4]   2> 1689973 T5544 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:51201/
   [junit4]   2> 1689974 T5544 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:51161/
   [junit4]   2> 1689974 T5544 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:51185/
   [junit4]   2> 1689974 T5544 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:51215/
   [junit4]   2> 1689975 T5544 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:51177/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=7739CFC0B2BE137A -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ja_JP_JP_#u-ca-japanese -Dtests.timezone=Asia/Phnom_Penh -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   81.4s J2 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:51177/c8n_1x3_commits_shard1_replica2
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([7739CFC0B2BE137A:FF6DF01A1C427E82]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 48 more
   [junit4]   2> 1689999 T5544 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7739CFC0B2BE137A-001
   [junit4]   2> 81357 T5543 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=DefaultSimilarity, locale=ja_JP_JP_#u-ca-japanese, timezone=Asia/Phnom_Penh
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=100946744,total=374865920
   [junit4]   2> NOTE: All tests run in this JVM: [LeaderElectionIntegrationTest, ParsingFieldUpdateProcessorsTest, DistributedFacetPivotLongTailTest, CachingDirectoryFactoryTest, TestTrie, TestMergePolicyConfig, TestMacros, TestDistribDocBasedVersion, TestFieldTypeCollectionResource, DistributedFacetPivotSmallTest, TestStressVersions, TestXIncludeConfig, DateFieldTest, DebugComponentTest, SpellingQueryConverterTest, AddSchemaFieldsUpdateProcessorFactoryTest, TestNoOpRegenerator, TestObjectReleaseTracker, CSVRequestHandlerTest, OutputWriterTest, TestConfigOverlay, TestReload, TestLRUStatsCache, TermsComponentTest, ReplicationFactorTest, CustomCollectionTest, TestIndexingPerformance, TestSolrIndexConfig, SpatialRPTFieldTypeTest, BasicDistributedZkTest, BadIndexSchemaTest, TestManagedResourceStorage, TestSearchPerf, TestCharFilters, TestLRUCache, SolrInfoMBeanTest, TestSolrDeletionPolicy1, TestDocBasedVersionConstraints, DeleteLastCustomShardedReplicaTest, LukeRequestHandlerTest, TestHashPartitioner, RecoveryZkTest, BadCopyFieldTest, QueryEqualityTest, TestRangeQuery, TestQuerySenderNoQuery, SchemaVersionSpecificBehaviorTest, TestHdfsUpdateLog, HighlighterMaxOffsetTest, TestDefaultStatsCache, TestDFRSimilarityFactory, RegexBoostProcessorTest, CollectionsAPIDistributedZkTest, DateMathParserTest, OverriddenZkACLAndCredentialsProvidersTest, TestFiltering, TestSimpleQParserPlugin, HighlighterConfigTest, HdfsUnloadDistributedZkTest, TestStressUserVersions, SyncSliceTest, TestCloudManagedSchemaConcurrent, UnloadDistributedZkTest, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J2 in 81.64s, 1 test, 1 error <<< FAILURES!

[...truncated 765 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 4 errors, 61 ignored (21 assumptions)

Total time: 57 minutes 1 second
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 218418265 bytes
Compression is 0.0%
Took 44 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2874 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2874/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:48476/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48476/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([E7E5C63DFB521D73:6FB1F9E755AE708B]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48337/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48337/collection1
	at __randomizedtesting.SeedInfo.seed([E7E5C63DFB521D73:6FB1F9E755AE708B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48337/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28719/jx/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28719/jx/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([E7E5C63DFB521D73:6FB1F9E755AE708B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28719/jx/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9372 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/init-core-data-001
   [junit4]   2> 210845 T861 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /jx/
   [junit4]   2> 210850 T861 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 210851 T862 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 210951 T861 oasc.ZkTestServer.run start zk server on port:25294
   [junit4]   2> 210952 T861 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 210953 T861 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 210957 T869 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@233730e0 name:ZooKeeperConnection Watcher:127.0.0.1:25294 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 210957 T861 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 210957 T861 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 210958 T861 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 210960 T861 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 210961 T861 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 210962 T872 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@69a681f4 name:ZooKeeperConnection Watcher:127.0.0.1:25294/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 210962 T861 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 210963 T861 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 210963 T861 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 210965 T861 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 210966 T861 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 210967 T861 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 210968 T861 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 210969 T861 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 210971 T861 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 210971 T861 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 210973 T861 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 210973 T861 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 210975 T861 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 210975 T861 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 210977 T861 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 210977 T861 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 210979 T861 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 210979 T861 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 210981 T861 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 210981 T861 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 210983 T861 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 210983 T861 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 210985 T861 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 210985 T861 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 210987 T861 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 210987 T861 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 210989 T861 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 210989 T861 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 212149 T861 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/cores/collection1
   [junit4]   2> 212153 T861 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 212157 T861 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:25300
   [junit4]   2> 212157 T861 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/jx, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/cores, hostPort=25299, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/tempDir-001/control/data}
   [junit4]   2> 212158 T861 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@5fb271f1
   [junit4]   2> 212158 T861 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/'
   [junit4]   2> 212199 T861 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/solr.xml
   [junit4]   2> 212250 T861 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/cores
   [junit4]   2> 212251 T861 oasc.CoreContainer.<init> New CoreContainer 1996701156
   [junit4]   2> 212251 T861 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/]
   [junit4]   2> 212251 T861 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/lib
   [junit4]   2> 212252 T861 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/lib).
   [junit4]   2> 212263 T861 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 212263 T861 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 212263 T861 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 212263 T861 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 212264 T861 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 212264 T861 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 212264 T861 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 212265 T861 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 212265 T861 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 212265 T861 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 212265 T861 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 212266 T861 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 212266 T861 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 212266 T861 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 212267 T861 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 212267 T861 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:25294/solr
   [junit4]   2> 212268 T861 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 212268 T861 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 212269 T861 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 212270 T884 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7204b839 name:ZooKeeperConnection Watcher:127.0.0.1:25294 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 212270 T861 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 212271 T861 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 212273 T861 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 212274 T887 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4fed9c09 name:ZooKeeperConnection Watcher:127.0.0.1:25294/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 212274 T861 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 212276 T861 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 212277 T861 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 212279 T861 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 212280 T861 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 212281 T861 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 212283 T861 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 212284 T861 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 212285 T861 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 212286 T861 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:25299_jx
   [junit4]   2> 212286 T861 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:25299_jx
   [junit4]   2> 212287 T861 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 212288 T861 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 212289 T861 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 212291 T861 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:25299_jx
   [junit4]   2> 212291 T861 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 212292 T861 oasc.Overseer.start Overseer (id=93581231638642691-127.0.0.1:25299_jx-n_0000000000) starting
   [junit4]   2> 212293 T861 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 212297 T861 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 212298 T889 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 212298 T861 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 212298 T888 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 212301 T861 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/cores
   [junit4]   2> 212303 T861 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {transient=false, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/cores/collection1, dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/cores/collection1/, config=solrconfig.xml, loadOnStartup=true, shard=, name=collection1, collection=control_collection, coreNodeName=}
   [junit4]   2> 212303 T861 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/cores/collection1/
   [junit4]   2> 212304 T861 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 212305 T891 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 212305 T891 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 212306 T891 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 212306 T887 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 212307 T888 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:25299_jx",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25299/jx"} current state version: 0
   [junit4]   2> 212307 T888 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:25299_jx",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25299/jx"}
   [junit4]   2> 212307 T888 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 212308 T888 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 212309 T887 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 213306 T891 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 213307 T891 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 213308 T891 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 213308 T891 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 213308 T891 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 213309 T891 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/cores/collection1/'
   [junit4]   2> 213323 T891 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 213325 T891 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 213337 T891 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 213357 T891 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 213411 T891 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 213412 T891 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 213430 T891 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 214234 T891 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 214267 T891 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 214269 T891 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 214311 T891 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 214315 T891 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 214318 T891 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 214322 T891 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 214323 T891 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 214323 T891 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 214324 T891 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 214324 T891 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 214333 T891 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 214334 T891 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 214334 T891 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/cores/collection1/data/
   [junit4]   2> 214334 T891 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4642ebd
   [junit4]   2> 214335 T891 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/cores/collection1/data
   [junit4]   2> 214335 T891 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/cores/collection1/data/index/
   [junit4]   2> 214335 T891 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 214336 T891 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/cores/collection1/data/index
   [junit4]   2> 214337 T891 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1061860050, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 214338 T891 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@830a434 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@316c561f),segFN=segments_1,generation=1}
   [junit4]   2> 214338 T891 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 214343 T891 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 214343 T891 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 214343 T891 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 214343 T891 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 214344 T891 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 214344 T891 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 214344 T891 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 214345 T891 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 214345 T891 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 214359 T891 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 214361 T891 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 214363 T891 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 214365 T891 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 214369 T891 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/threads,/get,/update/json,/admin/logging,/admin/ping,/admin/luke,/admin/file,/config,/admin/plugins,/admin/mbeans,/admin/properties,/schema,/admin/system,/update/json/docs,/admin/segments,standard,/update/csv,/replication,/update
   [junit4]   2> 214371 T891 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 214372 T891 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 214373 T891 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 214374 T891 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 214374 T891 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 214375 T891 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=353497832, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 214376 T891 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@830a434 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@316c561f),segFN=segments_1,generation=1}
   [junit4]   2> 214376 T891 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 214376 T891 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@442cd9fa[collection1] main
   [junit4]   2> 214376 T891 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 214377 T891 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 214377 T891 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 214378 T891 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 214378 T891 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 214378 T891 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 214379 T891 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 214379 T891 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 214379 T891 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 214380 T891 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 214380 T891 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 214380 T892 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@442cd9fa[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 214381 T895 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:25299/jx collection:control_collection shard:shard1
   [junit4]   2> 214381 T861 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 214382 T861 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 214382 T895 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 214385 T861 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 214385 T861 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 214387 T895 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 214387 T899 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1eb69805 name:ZooKeeperConnection Watcher:127.0.0.1:25294/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 214388 T861 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 214388 T861 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 214389 T861 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 214389 T887 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 214390 T895 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 214390 T895 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 214390 T888 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C112 name=collection1 org.apache.solr.core.SolrCore@27b6bc9e url=http://127.0.0.1:25299/jx/collection1 node=127.0.0.1:25299_jx C112_STATE=coll:control_collection core:collection1 props:{state=down, node_name=127.0.0.1:25299_jx, core=collection1, base_url=http://127.0.0.1:25299/jx}
   [junit4]   2> 214391 T895 C:control_collection S:shard1 C112 P25299 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:25299/jx/collection1/
   [junit4]   2> 214391 T895 C:control_collection S:shard1 C112 P25299 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 214392 T895 C:control_collection S:shard1 C112 P25299 oasc.SyncStrategy.syncToMe http://127.0.0.1:25299/jx/collection1/ has no replicas
   [junit4]   2> 214392 T861 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 214392 T887 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 214392 T899 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 214392 T895 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:25299/jx/collection1/ shard1
   [junit4]   2> 214394 T895 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 214397 T887 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 214398 T888 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25299/jx",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 214501 T887 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 214501 T899 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 214550 T895 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:25299/jx/collection1/ and leader is http://127.0.0.1:25299/jx/collection1/
   [junit4]   2> 214551 T895 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:25299/jx
   [junit4]   2> 214551 T895 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 214551 T895 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 214551 T895 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 214553 T887 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 214554 T888 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:25299_jx",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25299/jx"} current state version: 3
   [junit4]   2> 214554 T888 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:25299_jx",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25299/jx"}
   [junit4]   2> 214658 T887 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 214660 T899 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 216145 T861 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/cores/collection1
   [junit4]   2> 216148 T861 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001
   [junit4]   2> 216149 T861 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 216151 T861 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:60187
   [junit4]   2> 216151 T861 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=57218, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/cores, solrconfig=solrconfig.xml, hostContext=/jx, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/tempDir-001/jetty1}
   [junit4]   2> 216153 T861 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@5fb271f1
   [junit4]   2> 216153 T861 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/'
   [junit4]   2> 216190 T861 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/solr.xml
   [junit4]   2> 216236 T861 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/cores
   [junit4]   2> 216236 T861 oasc.CoreContainer.<init> New CoreContainer 1450341966
   [junit4]   2> 216237 T861 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/]
   [junit4]   2> 216237 T861 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/lib
   [junit4]   2> 216238 T861 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/lib).
   [junit4]   2> 216248 T861 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 216248 T861 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 216249 T861 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 216249 T861 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 216249 T861 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 216249 T861 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 216250 T861 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 216250 T861 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 216250 T861 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 216251 T861 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 216251 T861 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 216251 T861 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 216252 T861 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 216252 T861 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 216252 T861 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 216253 T861 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:25294/solr
   [junit4]   2> 216253 T861 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 216253 T861 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 216254 T861 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 216255 T911 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@189c091e name:ZooKeeperConnection Watcher:127.0.0.1:25294 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 216256 T861 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 216257 T861 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 216259 T861 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 216260 T914 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6c40cc11 name:ZooKeeperConnection Watcher:127.0.0.1:25294/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 216260 T861 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 216264 T861 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 217267 T861 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57218_jx
   [junit4]   2> 217268 T861 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:57218_jx
   [junit4]   2> 217271 T861 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 217273 T861 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/cores
   [junit4]   2> 217275 T861 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {loadOnStartup=true, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/cores/collection1, collection=collection1, config=solrconfig.xml, transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/cores/collection1/, shard=, coreNodeName=, dataDir=data/, name=collection1, schema=schema.xml}
   [junit4]   2> 217275 T861 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/cores/collection1/
   [junit4]   2> 217275 T861 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 217277 T915 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 217277 T915 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 217278 T915 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 217278 T887 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 217278 T888 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:57218_jx",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57218/jx"} current state version: 4
   [junit4]   2> 217279 T888 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:57218_jx",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57218/jx"}
   [junit4]   2> 217279 T888 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 217279 T888 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 217280 T887 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 217280 T899 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 217280 T914 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 218278 T915 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 218279 T915 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 218280 T915 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 218280 T915 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 218280 T915 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 218281 T915 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/cores/collection1/'
   [junit4]   2> 218296 T915 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 218299 T915 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 218313 T915 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 218336 T915 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 218390 T915 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 218392 T915 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 218411 T915 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 218777 T915 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 218793 T915 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 218795 T915 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 218808 T915 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 218813 T915 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 218817 T915 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 218818 T915 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 218818 T915 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 218818 T915 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 218819 T915 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 218819 T915 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 218819 T915 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 218820 T915 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 218820 T915 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 218820 T915 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4642ebd
   [junit4]   2> 218821 T915 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/cores/collection1/data
   [junit4]   2> 218821 T915 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 218822 T915 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 218822 T915 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 218823 T915 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1061860050, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 218824 T915 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@252e467c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@11589f6),segFN=segments_1,generation=1}
   [junit4]   2> 218824 T915 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 218842 T915 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 218843 T915 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 218843 T915 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 218843 T915 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 218844 T915 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 218844 T915 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 218844 T915 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 218845 T915 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 218845 T915 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 218867 T915 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 218869 T915 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 218871 T915 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 218874 T915 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 218880 T915 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/threads,/get,/update/json,/admin/logging,/admin/ping,/admin/luke,/admin/file,/config,/admin/plugins,/admin/mbeans,/admin/properties,/schema,/admin/system,/update/json/docs,/admin/segments,standard,/update/csv,/replication,/update
   [junit4]   2> 218882 T915 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 218884 T915 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 218884 T915 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 218885 T915 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 218886 T915 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 218887 T915 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=353497832, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 218888 T915 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@252e467c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@11589f6),segFN=segments_1,generation=1}
   [junit4]   2> 218888 T915 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 218888 T915 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@5bd26686[collection1] main
   [junit4]   2> 218889 T915 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 218890 T915 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 218890 T915 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 218890 T915 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 218890 T915 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 218891 T915 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 218891 T915 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 218891 T915 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 218892 T915 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 218892 T915 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 218893 T916 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5bd26686[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 218893 T915 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 218894 T919 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:57218/jx collection:collection1 shard:shard1
   [junit4]   2> 218895 T861 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 218895 T861 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 218895 T919 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 218899 T919 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 218900 T887 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 218901 T919 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 218901 T919 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C113 name=collection1 org.apache.solr.core.SolrCore@1f5e0b9b url=http://127.0.0.1:57218/jx/collection1 node=127.0.0.1:57218_jx C113_STATE=coll:collection1 core:collection1 props:{state=down, node_name=127.0.0.1:57218_jx, core=collection1, base_url=http://127.0.0.1:57218/jx}
   [junit4]   2> 218901 T919 C:collection1 S:shard1 C113 P57218 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:57218/jx/collection1/
   [junit4]   2> 218901 T888 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 218902 T919 C:collection1 S:shard1 C113 P57218 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 218902 T919 C:collection1 S:shard1 C113 P57218 oasc.SyncStrategy.syncToMe http://127.0.0.1:57218/jx/collection1/ has no replicas
   [junit4]   2> 218902 T919 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:57218/jx/collection1/ shard1
   [junit4]   2> 218903 T919 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 218903 T887 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 218903 T899 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 218903 T914 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 218906 T887 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 218907 T888 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57218/jx",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 219009 T887 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 219009 T914 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 219009 T899 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 219058 T919 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:57218/jx/collection1/ and leader is http://127.0.0.1:57218/jx/collection1/
   [junit4]   2> 219059 T919 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:57218/jx
   [junit4]   2> 219059 T919 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 219059 T919 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 219060 T919 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 219061 T887 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 219063 T888 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:57218_jx",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57218/jx"} current state version: 7
   [junit4]   2> 219064 T888 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:57218_jx",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57218/jx"}
   [junit4]   2> 219217 T887 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 219217 T914 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 219217 T899 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 220628 T861 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-2-001/cores/colle

[...truncated too long message...]

	... 11 more
   [junit4]   2> 	
   [junit4]   2> 568795 T1939 C:control_collection S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 568975 T1939 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:48453 48453
   [junit4]   2> 570027 T1940 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:48453 48453
   [junit4]   2> 570029 T1940 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 570029 T1939 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 1 connections to: http://127.0.0.1:48470/
   [junit4]   2> 570029 T1939 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:48487/
   [junit4]   2> 570030 T1939 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 9 connections to: http://127.0.0.1:48465/
   [junit4]   2> 570030 T1939 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:48459/
   [junit4]   2> 570031 T1939 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:48476/
   [junit4]   2> 570031 T1939 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:48470/
   [junit4]   2> 570031 T1939 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:48487/
   [junit4]   2> 570032 T1939 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:48465/
   [junit4]   2> 570032 T1939 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:48459/
   [junit4]   2> 570032 T1939 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:48476/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=E7E5C63DFB521D73 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=de_CH -Dtests.timezone=America/Grenada -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   78.0s J0 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48476/c8n_1x3_commits_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([E7E5C63DFB521D73:6FB1F9E755AE708B]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 48 more
   [junit4]   2> 570050 T1939 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E7E5C63DFB521D73-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=de_CH, timezone=America/Grenada
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=39416776,total=329777152
   [junit4]   2> NOTE: All tests run in this JVM: [BlockDirectoryTest, TestEmbeddedSolrServerConstructors, ReturnFieldsTest, TestSchemaNameResource, DistributedQueryComponentCustomSortTest, SpellPossibilityIteratorTest, FastVectorHighlighterTest, TestDistributedMissingSort, DistributedFacetPivotLongTailTest, TestCollectionAPI, HdfsWriteToMultipleCollectionsTest, TestHighFrequencyDictionaryFactory, TestSerializedLuceneMatchVersion, VMParamsZkACLAndCredentialsProvidersTest, StressHdfsTest, BasicZkTest, BinaryUpdateRequestHandlerTest, TestCoreContainer, MergeStrategyTest, TestCoreDiscovery, TestNonDefinedSimilarityFactory, TestSuggestSpellingConverter, TestFastWriter, TestFieldResource, CloudMLTQParserTest, TestRestManager, TestIndexSearcher, DistribDocExpirationUpdateProcessorTest, SolrRequestParserTest, TestLRUCache, CustomCollectionTest, TestMacros, AutoCommitTest, TestIndexingPerformance, TestQueryUtils, SolrCoreCheckLockOnStartupTest, FieldMutatingUpdateProcessorTest, DistributedDebugComponentTest, TestJettySolrRunner, OverseerRolesTest, TestSearcherReuse, TestComplexPhraseQParserPlugin, SpellingQueryConverterTest, HdfsCollectionsAPIDistributedZkTest, OutOfBoxZkACLAndCredentialsProvidersTest, TestHashPartitioner, DateMathParserTest, PingRequestHandlerTest, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J0 in 78.02s, 1 test, 1 error <<< FAILURES!

[...truncated 1172 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 53 minutes 40 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 218897246 bytes
Compression is 0.0%
Took 41 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2873 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2873/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:59968/dw_zba/ry/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:59968/dw_zba/ry/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([AA733A9D8B345833:2227054725C835CB]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:38410/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:38410/collection1
	at __randomizedtesting.SeedInfo.seed([AA733A9D8B345833:2227054725C835CB]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:38410/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:60922/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:60922/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([AA733A9D8B345833:2227054725C835CB]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:60922/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9707 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/init-core-data-001
   [junit4]   2> 495456 T4642 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 495464 T4642 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 495465 T4643 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 495565 T4642 oasc.ZkTestServer.run start zk server on port:38396
   [junit4]   2> 495566 T4642 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 495567 T4642 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 495570 T4650 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@fb05cc0 name:ZooKeeperConnection Watcher:127.0.0.1:38396 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 495571 T4642 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 495571 T4642 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 495571 T4642 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 495574 T4642 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 495575 T4642 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 495576 T4653 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3fd658e8 name:ZooKeeperConnection Watcher:127.0.0.1:38396/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 495576 T4642 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 495577 T4642 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 495577 T4642 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 495579 T4642 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 495581 T4642 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 495582 T4642 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 495584 T4642 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 495584 T4642 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 495587 T4642 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 495587 T4642 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 495690 T4642 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 495690 T4642 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 495692 T4642 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 495693 T4642 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 495695 T4642 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 495695 T4642 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 495697 T4642 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 495698 T4642 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 495700 T4642 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 495700 T4642 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 495702 T4642 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 495702 T4642 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 495704 T4642 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 495705 T4642 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 495707 T4642 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 495707 T4642 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 495709 T4642 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 495709 T4642 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 497069 T4642 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/cores/collection1
   [junit4]   2> 497074 T4642 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 497078 T4642 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:38400
   [junit4]   2> 497079 T4642 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/cores, hostPort=38399}
   [junit4]   2> 497080 T4642 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 497080 T4642 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/'
   [junit4]   2> 497130 T4642 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/solr.xml
   [junit4]   2> 497193 T4642 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/cores
   [junit4]   2> 497193 T4642 oasc.CoreContainer.<init> New CoreContainer 350067427
   [junit4]   2> 497194 T4642 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/]
   [junit4]   2> 497194 T4642 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/lib
   [junit4]   2> 497195 T4642 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/lib).
   [junit4]   2> 497209 T4642 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 497209 T4642 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 497209 T4642 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 497210 T4642 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 497210 T4642 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 497210 T4642 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 497211 T4642 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 497211 T4642 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 497211 T4642 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 497212 T4642 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 497212 T4642 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 497213 T4642 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 497213 T4642 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 497213 T4642 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 497214 T4642 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 497214 T4642 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:38396/solr
   [junit4]   2> 497214 T4642 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 497215 T4642 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 497216 T4642 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 497217 T4665 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@62bbd71c name:ZooKeeperConnection Watcher:127.0.0.1:38396 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 497218 T4642 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 497219 T4642 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 497221 T4642 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 497222 T4668 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2e1b0f94 name:ZooKeeperConnection Watcher:127.0.0.1:38396/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 497222 T4642 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 497224 T4642 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 497226 T4642 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 497228 T4642 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 497230 T4642 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 497232 T4642 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 497235 T4642 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 497236 T4642 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 497238 T4642 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 497239 T4642 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38399_
   [junit4]   2> 497239 T4642 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:38399_
   [junit4]   2> 497241 T4642 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 497242 T4642 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 497244 T4642 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 497245 T4642 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:38399_
   [junit4]   2> 497245 T4642 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 497247 T4642 oasc.Overseer.start Overseer (id=93580187931377667-127.0.0.1:38399_-n_0000000000) starting
   [junit4]   2> 497249 T4642 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 497255 T4642 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 497256 T4670 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 497256 T4642 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 497256 T4669 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 497260 T4642 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/cores
   [junit4]   2> 497261 T4642 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/cores/collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/cores/collection1/, collection=control_collection, loadOnStartup=true, config=solrconfig.xml, coreNodeName=, shard=, dataDir=data/, name=collection1, schema=schema.xml, transient=false}
   [junit4]   2> 497262 T4642 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/cores/collection1/
   [junit4]   2> 497262 T4642 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 497263 T4672 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 497263 T4672 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 497264 T4672 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 497264 T4668 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 497265 T4669 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38399",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:38399_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1"} current state version: 0
   [junit4]   2> 497265 T4669 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38399",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:38399_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 497266 T4669 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 497266 T4669 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 497267 T4668 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 498264 T4672 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 498265 T4672 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 498265 T4672 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 498266 T4672 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 498266 T4672 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 498266 T4672 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/cores/collection1/'
   [junit4]   2> 498279 T4672 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 498281 T4672 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 498290 T4672 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 498307 T4672 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 498346 T4672 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 498447 T4672 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 498456 T4672 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 498771 T4672 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 498786 T4672 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 498788 T4672 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 498797 T4672 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 498800 T4672 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 498804 T4672 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 498804 T4672 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 498804 T4672 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 498805 T4672 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 498805 T4672 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 498805 T4672 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 498806 T4672 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 498806 T4672 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 498806 T4672 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/cores/collection1/data/
   [junit4]   2> 498806 T4672 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@71e98cd2
   [junit4]   2> 498807 T4672 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/cores/collection1/data
   [junit4]   2> 498807 T4672 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/cores/collection1/data/index/
   [junit4]   2> 498807 T4672 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 498807 T4672 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/cores/collection1/data/index
   [junit4]   2> 498808 T4672 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=140968648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 498808 T4672 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5e5fcfbc lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@643f700b),segFN=segments_1,generation=1}
   [junit4]   2> 498809 T4672 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 498812 T4672 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 498812 T4672 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 498813 T4672 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 498813 T4672 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 498813 T4672 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 498813 T4672 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 498813 T4672 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 498814 T4672 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 498814 T4672 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 498833 T4672 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 498835 T4672 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 498837 T4672 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 498838 T4672 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 498842 T4672 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /get,/admin/ping,/admin/threads,/update/json/docs,/update/csv,/admin/mbeans,/config,/admin/properties,/admin/logging,/update,standard,/update/json,/schema,/admin/file,/admin/system,/admin/luke,/admin/plugins,/admin/segments,/replication
   [junit4]   2> 498844 T4672 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 498845 T4672 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 498845 T4672 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 498847 T4672 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 498847 T4672 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 498847 T4672 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=48, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.30719632747514725]
   [junit4]   2> 498848 T4672 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5e5fcfbc lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@643f700b),segFN=segments_1,generation=1}
   [junit4]   2> 498848 T4672 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 498848 T4672 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@6a43ab06[collection1] main
   [junit4]   2> 498849 T4672 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 498849 T4672 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 498849 T4672 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 498850 T4672 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 498850 T4672 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 498850 T4672 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 498850 T4672 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 498851 T4672 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 498851 T4672 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 498851 T4672 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 498851 T4672 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 498852 T4673 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6a43ab06[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 498852 T4676 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:38399 collection:control_collection shard:shard1
   [junit4]   2> 498852 T4642 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 498853 T4642 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 498853 T4676 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 498855 T4642 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 498856 T4642 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 498857 T4676 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 498857 T4680 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@cbf4ff2 name:ZooKeeperConnection Watcher:127.0.0.1:38396/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 498857 T4642 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 498858 T4642 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 498858 T4642 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 498858 T4668 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 498859 T4676 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 498859 T4676 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C637 name=collection1 org.apache.solr.core.SolrCore@4fe7d389 url=http://127.0.0.1:38399/collection1 node=127.0.0.1:38399_ C637_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:38399, node_name=127.0.0.1:38399_, core=collection1, state=down}
   [junit4]   2> 498859 T4676 C:control_collection S:shard1 C637 P38399 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:38399/collection1/
   [junit4]   2> 498859 T4676 C:control_collection S:shard1 C637 P38399 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 498859 T4669 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 498859 T4676 C:control_collection S:shard1 C637 P38399 oasc.SyncStrategy.syncToMe http://127.0.0.1:38399/collection1/ has no replicas
   [junit4]   2> 498860 T4676 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:38399/collection1/ shard1
   [junit4]   2> 498860 T4676 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 498860 T4668 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 498860 T4680 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 498861 T4642 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 498863 T4668 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 498864 T4669 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38399",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 498967 T4668 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 498967 T4680 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 499016 T4676 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:38399/collection1/ and leader is http://127.0.0.1:38399/collection1/
   [junit4]   2> 499017 T4676 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:38399
   [junit4]   2> 499017 T4676 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 499017 T4676 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 499017 T4676 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 499018 T4668 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 499019 T4669 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38399",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:38399_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1"} current state version: 3
   [junit4]   2> 499020 T4669 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38399",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:38399_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 499122 T4680 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 499122 T4668 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 500146 T4642 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/cores/collection1
   [junit4]   2> 500149 T4642 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001
   [junit4]   2> 500150 T4642 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 500152 T4642 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:38411
   [junit4]   2> 500152 T4642 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, hostPort=38410, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/tempDir-001/jetty1, solrconfig=solrconfig.xml}
   [junit4]   2> 500153 T4642 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 500153 T4642 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/'
   [junit4]   2> 500200 T4642 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/solr.xml
   [junit4]   2> 500259 T4642 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/cores
   [junit4]   2> 500260 T4642 oasc.CoreContainer.<init> New CoreContainer 743233151
   [junit4]   2> 500260 T4642 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/]
   [junit4]   2> 500260 T4642 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/lib
   [junit4]   2> 500261 T4642 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/lib).
   [junit4]   2> 500271 T4642 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 500271 T4642 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 500271 T4642 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 500271 T4642 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 500272 T4642 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 500272 T4642 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 500272 T4642 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 500272 T4642 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 500273 T4642 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 500273 T4642 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 500273 T4642 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 500273 T4642 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 500274 T4642 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 500274 T4642 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 500274 T4642 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 500274 T4642 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:38396/solr
   [junit4]   2> 500275 T4642 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 500275 T4642 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 500275 T4642 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 500277 T4692 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@710e9427 name:ZooKeeperConnection Watcher:127.0.0.1:38396 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 500277 T4642 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 500278 T4642 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 500280 T4642 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 500281 T4695 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5cbfb5cb name:ZooKeeperConnection Watcher:127.0.0.1:38396/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 500281 T4642 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 500285 T4642 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 501289 T4642 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38410_
   [junit4]   2> 501290 T4642 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:38410_
   [junit4]   2> 501292 T4642 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 501294 T4642 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/cores
   [junit4]   2> 501297 T4642 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {dataDir=data/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/cores/collection1, coreNodeName=, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/cores/collection1/, collection=collection1, schema=schema.xml, loadOnStartup=true, name=collection1, transient=false, shard=}
   [junit4]   2> 501297 T4642 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/cores/collection1/
   [junit4]   2> 501297 T4642 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 501298 T4696 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 501298 T4696 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 501299 T4668 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 501299 T4696 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 501299 T4669 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38410",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:38410_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1"} current state version: 4
   [junit4]   2> 501300 T4669 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38410",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:38410_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 501300 T4669 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 501300 T4669 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 501301 T4695 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 501301 T4680 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 501301 T4668 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 502299 T4696 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 502300 T4696 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 502301 T4696 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 502301 T4696 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 502302 T4696 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 502302 T4696 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/cores/collection1/'
   [junit4]   2> 502314 T4696 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 502316 T4696 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 502325 T4696 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 502341 T4696 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 502403 T4696 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 502505 T4696 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 502514 T4696 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 502833 T4696 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 502854 T4696 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 502857 T4696 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 502869 T4696 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 502874 T4696 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 502878 T4696 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 502879 T4696 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 502880 T4696 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 502880 T4696 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 502881 T4696 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 502881 T4696 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 502881 T4696 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 502881 T4696 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 502882 T4696 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 502882 T4696 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@71e98cd2
   [junit4]   2> 502883 T4696 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/cores/collection1/data
   [junit4]   2> 502883 T4696 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 502883 T4696 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 502884 T4696 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 502885 T4696 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=140968648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 502886 T4696 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@70fec487 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6618b505),segFN=segments_1,generation=1}
   [junit4]   2> 502886 T4696 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 502892 T4696 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 502893 T4696 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 502893 T4696 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 502893 T4696 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 502893 T4696 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 502894 T4696 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 502894 T4696 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 502895 T4696 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 502895 T4696 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 502917 T4696 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 502919 T4696 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 502922 T4696 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 502925 T4696 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 502933 T4696 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /get,/admin/ping,/admin/threads,/update/json/docs,/update/csv,/admin/mbeans,/config,/admin/properties,/admin/logging,/update,standard,/update/json,/schema,/admin/file,/admin/system,/admin/luke,/admin/plugins,/admin/segments,/replication
   [junit4]   2> 502935 T4696 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 502936 T4696 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 502937 T4696 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 502938 T4696 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 502939 T4696 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 502940 T4696 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=48, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.30719632747514725]
   [junit4]   2> 502940 T4696 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@70fec487 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6618b505),segFN=segments_1,generation=1}
   [junit4]   2> 502940 T4696 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 502941 T4696 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@14dfd754[collection1] main
   [junit4]   2> 502941 T4696 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 502942 T4696 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 502943 T4696 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 502943 T4696 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 502943 T4696 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 502943 T4696 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 502944 T4696 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 502944 T4696 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 502945 T4696 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 502945 T4696 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 502946 T4697 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@14dfd754[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 502946 T4696 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 502947 T4700 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:38410 collection:collection1 shard:shard1
   [junit4]   2> 502947 T4642 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 502948 T4642 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 502948 T4700 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 502953 T4700 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 502955 T4668 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 502955 T4700 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 502955 T4700 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C638 name=collection1 org.apache.solr.core.SolrCore@5a0039e7 url=http://127.0.0.1:38410/collection1 node=127.0.0.1:38410_ C638_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:38410, node_name=127.0.0.1:38410_, core=collection1, state=down}
   [junit4]   2> 502956 T4700 C:collection1 S:shard1 C638 P38410 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:38410/collection1/
   [junit4]   2> 502956 T4700 C:collection1 S:shard1 C638 P38410 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 502956 T4669 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 502956 T4700 C:collection1 S:shard1 C638 P38410 oasc.SyncStrategy.syncToMe http://127.0.0.1:38410/collection1/ has no replicas
   [junit4]   2> 502956 T4700 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:38410/collection1/ shard1
   [junit4]   2> 502956 T4700 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 502957 T4680 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 502958 T4695 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 502958 T4668 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 502962 T4669 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38410",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 502964 T4668 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 502965 T4680 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 502965 T4695 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 502965 T4668 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 503012 T4700 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:38410/collection1/ and leader is http://127.0.0.1:38410/collection1/
   [junit4]   2> 503013 T4700 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:38410
   [junit4]   2> 503013 T4700 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 503013 T4700 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 503013 T4700 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 503015 T4668 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 503016 T4669 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38410",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:38410_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1"} current state version: 7
   [junit4]   2> 503017 T4669 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38410",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:38410_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 503121 T4680 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 503121 T4695 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDa

[...truncated too long message...]

it4]   2> 	
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent data watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		2	/solr/collections/c8n_1x3_commits/leader_elect/shard1/election/93580244919779340-core_node3-n_0000000000
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1444103 T3058 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:59968/dw_zba/ry
   [junit4]   2> 1444103 T3058 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 3 connections to: http://127.0.0.1:59877/dw_zba/ry
   [junit4]   2> 1444103 T3058 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:59993/dw_zba/ry
   [junit4]   2> 1444103 T3058 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 11 connections to: http://127.0.0.1:59886/dw_zba/ry
   [junit4]   2> 1444104 T3058 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 1 connections to: http://127.0.0.1:59938/dw_zba/ry
   [junit4]   2> 1444104 T3058 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:59968/dw_zba/ry
   [junit4]   2> 1444104 T3058 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:59877/dw_zba/ry
   [junit4]   2> 1444105 T3058 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:59993/dw_zba/ry
   [junit4]   2> 1444105 T3058 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:59886/dw_zba/ry
   [junit4]   2> 1444105 T3058 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:59938/dw_zba/ry
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=AA733A9D8B345833 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=mt_MT -Dtests.timezone=Canada/Eastern -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   78.8s J1 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:59968/dw_zba/ry/c8n_1x3_commits_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([AA733A9D8B345833:2227054725C835CB]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 48 more
   [junit4]   2> 1444126 T3058 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest AA733A9D8B345833-001
   [junit4]   2> 78834 T3057 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=true,coord=no): {}, locale=mt_MT, timezone=Canada/Eastern
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=84144608,total=299368448
   [junit4]   2> NOTE: All tests run in this JVM: [ExitableDirectoryReaderTest, TestCodecSupport, FastVectorHighlighterTest, IndexBasedSpellCheckerTest, TestCopyFieldCollectionResource, RegexBoostProcessorTest, BasicZkTest, CloudMLTQParserTest, ChaosMonkeyNothingIsSafeTest, SearchHandlerTest, TestInfoStreamLogging, TestXIncludeConfig, ScriptEngineTest, TestNRTOpen, SynonymTokenizerTest, TestConfigSets, TestSolrDynamicMBean, CopyFieldTest, MultiTermTest, URLClassifyProcessorTest, TestBulkSchemaAPI, TestBM25SimilarityFactory, ZkCLITest, TestZkChroot, TestSchemaSimilarityResource, OpenCloseCoreStressTest, DocumentBuilderTest, UpdateParamsTest, DistributedFacetPivotLargeTest, DistributedFacetPivotSmallAdvancedTest, TestPostingsSolrHighlighter, UniqFieldsUpdateProcessorFactoryTest, TestDocumentBuilder, UnloadDistributedZkTest, IndexSchemaTest, DateFieldTest, ExternalCollectionsTest, TestStressRecovery, NoCacheHeaderTest, TestJmxMonitoredMap, MBeansHandlerTest, SystemInfoHandlerTest, ResourceLoaderTest, TestJsonRequest, TestFastLRUCache, TestNamedUpdateProcessors, SimpleCollectionCreateDeleteTest, SpellingQueryConverterTest, TestBulkSchemaConcurrent, LoggingHandlerTest, TestLeaderElectionZkExpiry, RequestHandlersTest, BasicFunctionalityTest, SolrCoreTest, TestSerializedLuceneMatchVersion, TestDocSet, VMParamsZkACLAndCredentialsProvidersTest, LeaderFailoverAfterPartitionTest, BasicDistributedZk2Test, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J1 in 79.62s, 1 test, 1 error <<< FAILURES!

[...truncated 719 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 51 minutes 57 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 214555497 bytes
Compression is 0.0%
Took 37 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2872 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2872/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:46386/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:46386/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([7319EB1FBFFBB865:FB4DD4C51107D59D]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:43189/_dr/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:43189/_dr/collection1
	at __randomizedtesting.SeedInfo.seed([7319EB1FBFFBB865:FB4DD4C51107D59D]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:43189/_dr/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:57081/repfacttest_c8n_1x3_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:57081/repfacttest_c8n_1x3_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([7319EB1FBFFBB865:FB4DD4C51107D59D]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:57081/repfacttest_c8n_1x3_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9657 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/init-core-data-001
   [junit4]   2> 650391 T1557 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 650396 T1557 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 650397 T1558 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 650497 T1557 oasc.ZkTestServer.run start zk server on port:16356
   [junit4]   2> 650498 T1557 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 650499 T1557 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 650503 T1565 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@666b8451 name:ZooKeeperConnection Watcher:127.0.0.1:16356 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 650503 T1557 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 650504 T1557 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 650504 T1557 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 650507 T1557 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 650508 T1557 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 650510 T1568 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4ed528fe name:ZooKeeperConnection Watcher:127.0.0.1:16356/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 650510 T1557 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 650511 T1557 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 650511 T1557 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 650514 T1557 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 650515 T1557 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 650516 T1557 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 650517 T1557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 650518 T1557 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 650520 T1557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 650521 T1557 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 650622 T1557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 650623 T1557 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 650624 T1557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 650625 T1557 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 650626 T1557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 650626 T1557 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 650628 T1557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 650628 T1557 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 650648 T1557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 650648 T1557 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 650651 T1557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 650652 T1557 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 650657 T1557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 650658 T1557 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 650660 T1557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 650660 T1557 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 650661 T1557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 650662 T1557 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 651604 T1557 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/cores/collection1
   [junit4]   2> 651608 T1557 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 651612 T1557 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:17992
   [junit4]   2> 651612 T1557 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/tempDir-001/control/data, hostPort=63033}
   [junit4]   2> 651613 T1557 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 651613 T1557 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/'
   [junit4]   2> 651643 T1557 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/solr.xml
   [junit4]   2> 651678 T1557 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/cores
   [junit4]   2> 651678 T1557 oasc.CoreContainer.<init> New CoreContainer 566788789
   [junit4]   2> 651678 T1557 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/]
   [junit4]   2> 651678 T1557 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/lib
   [junit4]   2> 651679 T1557 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/lib).
   [junit4]   2> 651688 T1557 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 651688 T1557 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 651688 T1557 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 651689 T1557 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 651689 T1557 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 651689 T1557 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 651689 T1557 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 651689 T1557 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 651690 T1557 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 651690 T1557 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 651690 T1557 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 651690 T1557 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 651691 T1557 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 651691 T1557 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 651691 T1557 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 651691 T1557 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:16356/solr
   [junit4]   2> 651692 T1557 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 651692 T1557 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 651693 T1557 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 651694 T1580 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7b435b90 name:ZooKeeperConnection Watcher:127.0.0.1:16356 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 651695 T1557 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 651696 T1557 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 651698 T1557 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 651699 T1583 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6c6b8fd4 name:ZooKeeperConnection Watcher:127.0.0.1:16356/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 651700 T1557 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 651701 T1557 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 651703 T1557 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 651705 T1557 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 651707 T1557 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 651708 T1557 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 651710 T1557 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 651711 T1557 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 651713 T1557 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 651714 T1557 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63033_
   [junit4]   2> 651714 T1557 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:63033_
   [junit4]   2> 651715 T1557 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 651717 T1557 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 651718 T1557 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 651719 T1557 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:63033_
   [junit4]   2> 651720 T1557 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 651721 T1557 oasc.Overseer.start Overseer (id=93579341981417475-127.0.0.1:63033_-n_0000000000) starting
   [junit4]   2> 651722 T1557 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 651727 T1557 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 651728 T1585 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 651728 T1557 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 651728 T1584 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 651732 T1557 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/cores
   [junit4]   2> 651734 T1557 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=control_collection, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/cores/collection1, name=collection1, transient=false, dataDir=data/, shard=, loadOnStartup=true, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/cores/collection1/, schema=schema.xml}
   [junit4]   2> 651734 T1557 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/cores/collection1/
   [junit4]   2> 651734 T1557 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 651735 T1587 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 651735 T1587 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 651736 T1587 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 651736 T1583 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 651737 T1584 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:63033",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:63033_",
   [junit4]   2> 	  "core":"collection1"} current state version: 0
   [junit4]   2> 651737 T1584 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:63033",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:63033_",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 651738 T1584 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 651738 T1584 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 651739 T1583 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 652736 T1587 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 652737 T1587 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 652737 T1587 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 652738 T1587 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 652738 T1587 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 652739 T1587 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/cores/collection1/'
   [junit4]   2> 652757 T1587 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 652759 T1587 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 652770 T1587 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 652792 T1587 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 652855 T1587 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 652956 T1587 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 652967 T1587 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 653375 T1587 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 653392 T1587 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 653395 T1587 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 653407 T1587 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 653412 T1587 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 653416 T1587 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 653417 T1587 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 653417 T1587 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 653418 T1587 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 653418 T1587 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 653419 T1587 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 653419 T1587 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 653419 T1587 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 653420 T1587 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/cores/collection1/data/
   [junit4]   2> 653420 T1587 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6fd80308
   [junit4]   2> 653421 T1587 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/cores/collection1/data
   [junit4]   2> 653421 T1587 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/cores/collection1/data/index/
   [junit4]   2> 653421 T1587 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 653422 T1587 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/cores/collection1/data/index
   [junit4]   2> 653422 T1587 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=37, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 653423 T1587 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@62271b45 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@756f815a),segFN=segments_1,generation=1}
   [junit4]   2> 653423 T1587 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 653430 T1587 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 653430 T1587 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 653431 T1587 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 653431 T1587 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 653431 T1587 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 653431 T1587 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 653432 T1587 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 653432 T1587 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 653433 T1587 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 653452 T1587 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 653454 T1587 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 653456 T1587 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 653458 T1587 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 653464 T1587 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: standard,/admin/file,/update/csv,/get,/update,/admin/plugins,/schema,/update/json/docs,/admin/ping,/admin/properties,/admin/logging,/admin/mbeans,/admin/system,/admin/segments,/config,/replication,/admin/threads,/admin/luke,/update/json
   [junit4]   2> 653467 T1587 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 653468 T1587 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 653468 T1587 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 653470 T1587 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 653470 T1587 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 653471 T1587 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=46, maxMergeAtOnceExplicit=49, maxMergedSegmentMB=70.00390625, floorSegmentMB=1.810546875, forceMergeDeletesPctAllowed=17.496924637359008, segmentsPerTier=16.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.19937087188091374
   [junit4]   2> 653471 T1587 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@62271b45 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@756f815a),segFN=segments_1,generation=1}
   [junit4]   2> 653472 T1587 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 653472 T1587 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@2a712cf5[collection1] main
   [junit4]   2> 653472 T1587 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 653473 T1587 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 653474 T1587 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 653474 T1587 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 653474 T1587 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 653474 T1587 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 653475 T1587 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 653475 T1587 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 653476 T1587 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 653476 T1587 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 653477 T1588 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2a712cf5[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 653477 T1587 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 653478 T1591 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:63033 collection:control_collection shard:shard1
   [junit4]   2> 653478 T1557 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 653479 T1557 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 653479 T1591 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 653482 T1557 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 653483 T1557 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 653483 T1591 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 653484 T1595 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@142d0aca name:ZooKeeperConnection Watcher:127.0.0.1:16356/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 653484 T1557 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 653485 T1583 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 653485 T1591 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 653485 T1557 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 653485 T1584 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 653485 T1557 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 653485 T1591 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C274 name=collection1 org.apache.solr.core.SolrCore@37ffd969 url=http://127.0.0.1:63033/collection1 node=127.0.0.1:63033_ C274_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:63033, state=down, node_name=127.0.0.1:63033_, core=collection1}
   [junit4]   2> 653487 T1591 C:control_collection S:shard1 C274 P63033 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:63033/collection1/
   [junit4]   2> 653487 T1591 C:control_collection S:shard1 C274 P63033 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 653487 T1583 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 653487 T1591 C:control_collection S:shard1 C274 P63033 oasc.SyncStrategy.syncToMe http://127.0.0.1:63033/collection1/ has no replicas
   [junit4]   2> 653488 T1591 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:63033/collection1/ shard1
   [junit4]   2> 653487 T1595 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 653488 T1591 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 653489 T1557 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 653492 T1583 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 653492 T1584 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:63033",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 653595 T1583 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 653596 T1595 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 653644 T1591 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:63033/collection1/ and leader is http://127.0.0.1:63033/collection1/
   [junit4]   2> 653645 T1591 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:63033
   [junit4]   2> 653645 T1591 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 653645 T1591 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 653645 T1591 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 653647 T1583 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 653647 T1584 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:63033",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:63033_",
   [junit4]   2> 	  "core":"collection1"} current state version: 3
   [junit4]   2> 653648 T1584 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:63033",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:63033_",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 653751 T1583 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 653751 T1595 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 654728 T1557 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/cores/collection1
   [junit4]   2> 654730 T1557 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001
   [junit4]   2> 654731 T1557 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 654733 T1557 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:20037
   [junit4]   2> 654734 T1557 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/tempDir-001/jetty1, hostPort=46386, solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/cores}
   [junit4]   2> 654734 T1557 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 654734 T1557 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/'
   [junit4]   2> 654764 T1557 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/solr.xml
   [junit4]   2> 654799 T1557 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/cores
   [junit4]   2> 654800 T1557 oasc.CoreContainer.<init> New CoreContainer 1960725163
   [junit4]   2> 654800 T1557 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/]
   [junit4]   2> 654800 T1557 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/lib
   [junit4]   2> 654801 T1557 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/lib).
   [junit4]   2> 654811 T1557 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 654811 T1557 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 654811 T1557 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 654811 T1557 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 654812 T1557 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 654812 T1557 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 654812 T1557 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 654812 T1557 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 654812 T1557 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 654813 T1557 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 654813 T1557 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 654813 T1557 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 654814 T1557 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 654814 T1557 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 654814 T1557 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 654814 T1557 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:16356/solr
   [junit4]   2> 654814 T1557 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 654815 T1557 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 654815 T1557 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 654817 T1607 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@710ed421 name:ZooKeeperConnection Watcher:127.0.0.1:16356 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 654818 T1557 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 654818 T1557 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 654821 T1557 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 654822 T1610 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@272f38c3 name:ZooKeeperConnection Watcher:127.0.0.1:16356/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 654822 T1557 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 654826 T1557 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 655829 T1557 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46386_
   [junit4]   2> 655830 T1557 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:46386_
   [junit4]   2> 655833 T1557 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 655836 T1557 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/cores
   [junit4]   2> 655839 T1557 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/cores/collection1/, coreNodeName=, name=collection1, schema=schema.xml, dataDir=data/, transient=false, collection=collection1, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/cores/collection1, loadOnStartup=true, shard=}
   [junit4]   2> 655839 T1557 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/cores/collection1/
   [junit4]   2> 655840 T1557 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 655841 T1611 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 655841 T1611 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 655842 T1611 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 655842 T1583 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 655843 T1584 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:46386",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:46386_",
   [junit4]   2> 	  "core":"collection1"} current state version: 4
   [junit4]   2> 655843 T1584 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:46386",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:46386_",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 655843 T1584 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 655844 T1584 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 655845 T1583 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 655845 T1610 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 655845 T1595 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 656842 T1611 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 656843 T1611 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 656843 T1611 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 656844 T1611 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 656844 T1611 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 656844 T1611 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/cores/collection1/'
   [junit4]   2> 656857 T1611 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 656859 T1611 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 656867 T1611 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 656882 T1611 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 656916 T1611 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 657017 T1611 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 657024 T1611 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 657415 T1611 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 657427 T1611 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 657429 T1611 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 657438 T1611 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 657442 T1611 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 657444 T1611 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 657445 T1611 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 657445 T1611 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 657446 T1611 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 657446 T1611 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 657446 T1611 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 657447 T1611 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 657447 T1611 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 657447 T1611 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 657447 T1611 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6fd80308
   [junit4]   2> 657448 T1611 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/cores/collection1/data
   [junit4]   2> 657448 T1611 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 657448 T1611 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 657449 T1611 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 657449 T1611 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=37, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 657449 T1611 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@bec9346 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@58b8dec7),segFN=segments_1,generation=1}
   [junit4]   2> 657450 T1611 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 657455 T1611 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 657456 T1611 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 657456 T1611 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 657456 T1611 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 657456 T1611 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 657456 T1611 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 657457 T1611 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 657457 T1611 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 657457 T1611 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 657469 T1611 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 657471 T1611 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 657472 T1611 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 657474 T1611 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 657480 T1611 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: standard,/admin/file,/update/csv,/get,/update,/admin/plugins,/schema,/update/json/docs,/admin/ping,/admin/properties,/admin/logging,/admin/mbeans,/admin/system,/admin/segments,/config,/replication,/admin/threads,/admin/luke,/update/json
   [junit4]   2> 657482 T1611 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 657483 T1611 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 657483 T1611 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 657484 T1611 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 657484 T1611 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 657485 T1611 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=46, maxMergeAtOnceExplicit=49, maxMergedSegmentMB=70.00390625, floorSegmentMB=1.810546875, forceMergeDeletesPctAllowed=17.496924637359008, segmentsPerTier=16.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.19937087188091374
   [junit4]   2> 657486 T1611 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@bec9346 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@58b8dec7),segFN=segments_1,generation=1}
   [junit4]   2> 657486 T1611 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 657487 T1611 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@33f8cc90[collection1] main
   [junit4]   2> 657487 T1611 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 657488 T1611 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 657488 T1611 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 657488 T1611 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 657488 T1611 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 657488 T1611 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 657489 T1611 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 657489 T1611 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 657489 T1611 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 657489 T1611 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 657491 T1612 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@33f8cc90[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 657491 T1611 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 657492 T1615 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:46386 collection:collection1 shard:shard1
   [junit4]   2> 657492 T1557 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 657493 T1557 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 657493 T1615 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 657497 T1615 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 657498 T1583 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 657498 T1615 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 657498 T1615 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C275 name=collection1 org.apache.solr.core.SolrCore@3b4dc481 url=http://127.0.0.1:46386/collection1 node=127.0.0.1:46386_ C275_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:46386, state=down, node_name=127.0.0.1:46386_, core=collection1}
   [junit4]   2> 657498 T1615 C:collection1 S:shard1 C275 P46386 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:46386/collection1/
   [junit4]   2> 657499 T1584 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 657499 T1615 C:collection1 S:shard1 C275 P46386 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 657499 T1615 C:collection1 S:shard1 C275 P46386 oasc.SyncStrategy.syncToMe http://127.0.0.1:46386/collection1/ has no replicas
   [junit4]   2> 657499 T1615 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:46386/collection1/ shard1
   [junit4]   2> 657500 T1615 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 657500 T1583 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 657500 T1610 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 657500 T1595 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 657503 T1583 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 657504 T1584 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:46386",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 657607 T1610 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 657607 T1595 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 657607 T1583 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 657656 T1615 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:46386/collection1/ and leader is http://127.0.0.1:46386/collection1/
   [junit4]   2> 657687 T1615 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:46386
   [junit4]   2> 657688 T1615 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 657688 T1615 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 657688 T1615 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 657690 T1583 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 657691 T1584 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:46386",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:46386_",
   [junit4]   2> 	  "core":"collection1"} current state version: 7
   [junit4]   2> 657691 T1584 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:46386",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:46386_",
   [junit4]   2> 	  "c

[...truncated too long message...]

tion S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/_dr,null}
   [junit4]   2> 1710172 T4651 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:43174 43174
   [junit4]   2> 1710392 T4652 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:43174 43174
   [junit4]   2> 1710393 T4652 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1710394 T4651 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:43207/_dr
   [junit4]   2> 1710394 T4651 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:43189/_dr
   [junit4]   2> 1710394 T4651 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:43180/_dr
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=7319EB1FBFFBB865 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=vi -Dtests.timezone=America/Managua -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   68.3s J3 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:43189/_dr/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([7319EB1FBFFBB865:FB4DD4C51107D59D]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:43189/_dr/collection1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 54 more
   [junit4]   2> 1710410 T4651 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7319EB1FBFFBB865-001
   [junit4]   2> 68280 T4650 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50), sim=RandomSimilarityProvider(queryNorm=true,coord=no): {}, locale=vi, timezone=America/Managua
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=93747856,total=325582848
   [junit4]   2> NOTE: All tests run in this JVM: [NumericFieldsTest, SuggesterTSTTest, TestNamedUpdateProcessors, DistributedExpandComponentTest, TestHashPartitioner, BasicDistributedZk2Test, RankQueryTest, TestIntervalFaceting, TestConfigOverlay, TestDefaultSearchFieldResource, DeleteInactiveReplicaTest, TestMissingGroups, CurrencyFieldOpenExchangeTest, BlockCacheTest, XsltUpdateRequestHandlerTest, AnalyticsQueryTest, FileBasedSpellCheckerTest, TestWordDelimiterFilterFactory, MigrateRouteKeyTest, HdfsBasicDistributedZkTest, TestExtendedDismaxParser, PingRequestHandlerTest, TestNonDefinedSimilarityFactory, TestExceedMaxTermLength, QueryResultKeyTest, LeaderInitiatedRecoveryOnCommitTest, TestJsonRequest, FastVectorHighlighterTest, TestQuerySenderListener, IndexBasedSpellCheckerTest, TestFieldTypeResource, TestPivotHelperCode, ExternalCollectionsTest, TestRangeQuery, TestXIncludeConfig, ZkSolrClientTest, HdfsDirectoryFactoryTest, SpellingQueryConverterTest, TestJmxIntegration, TestExactSharedStatsCache, TestIndexingPerformance, HighlighterTest, SOLR749Test, UniqFieldsUpdateProcessorFactoryTest, LukeRequestHandlerTest, TestSearchPerf, XmlUpdateRequestHandlerTest, TestDistributedMissingSort, TestReload, DirectSolrConnectionTest, SolrXmlInZkTest, TestArbitraryIndexDir, TestBulkSchemaConcurrent, TestAddFieldRealTimeGet, HdfsChaosMonkeySafeLeaderTest, SyncSliceTest, ShardSplitTest, FullSolrCloudDistribCmdsTest, TestSolrConfigHandlerConcurrent, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J3 in 68.56s, 1 test, 1 error <<< FAILURES!

[...truncated 689 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 55 minutes 8 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 224382140 bytes
Compression is 0.0%
Took 40 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure